learnware/examples/dataset_table_workflow/homo.py

import os
import warnings
import numpy as np
warnings.filterwarnings("ignore")

from learnware.market import BaseUserInfo
from learnware.logger import get_module_logger
from learnware.specification import generate_stat_spec
from learnware.reuse import AveragingReuser, JobSelectorReuser, EnsemblePruningReuser

from methods import *
from base import TableWorkflow
from config import n_labeled_list
from utils import Recorder, plot_performance_curves

logger = get_module_logger("homo_table", level="INFO")


class CorporacionDatasetWorkflow(TableWorkflow):
    def unlabeled_homo_table_example(self):
        logger.info("Total Item: %d" % (len(self.market)))
        learnware_rmse_list = []
        single_score_list = []
        job_selector_score_list = []
        ensemble_score_list = []
        pruning_score_list = []
        all_learnwares = self.market.get_learnwares()

        user = self.benchmark.name
        for idx in range(self.benchmark.user_num):
            test_x, test_y = self.benchmark.get_test_data(user_ids=idx)
            test_x, test_y = test_x.values, test_y.values
            user_stat_spec = generate_stat_spec(type="table", X=test_x)
            user_info = BaseUserInfo(
                semantic_spec=self.user_semantic, stat_info={user_stat_spec.type: user_stat_spec}
            )
            logger.info(f"Searching Market for user: {user}_{idx}")

            search_result = self.market.search_learnware(user_info)
            single_result = search_result.get_single_results()
            multiple_result = search_result.get_multiple_results()

            logger.info(f"search result of user {user}_{idx}:")
            logger.info(
                f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
            )
            
            pred_y = single_result[0].learnware.predict(test_x)
            single_score_list.append(loss_func_rmse(pred_y, test_y))

            rmse_list = []
            for learnware in all_learnwares:
                pred_y = learnware.predict(test_x)
                rmse_list.append(loss_func_rmse(pred_y, test_y))
            logger.info(
                f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, rmse: {single_score_list[-1]}"
            )

            if len(multiple_result) > 0:
                mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
                logger.info(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
                mixture_learnware_list = multiple_result[0].learnwares
            else:
                mixture_learnware_list = [single_result[0].learnware]

            # test reuse (job selector)
            reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list, herding_num=100)
            reuse_predict = reuse_baseline.predict(user_data=test_x)
            reuse_score = loss_func_rmse(reuse_predict, test_y)
            job_selector_score_list.append(reuse_score)
            logger.info(f"mixture reuse rmse (job selector): {reuse_score}")

            # test reuse (ensemble)
            reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="mean")
            ensemble_predict_y = reuse_ensemble.predict(user_data=test_x)
            ensemble_score = loss_func_rmse(ensemble_predict_y, test_y)
            ensemble_score_list.append(ensemble_score)
            logger.info(f"mixture reuse rmse (ensemble): {ensemble_score}")

            # test reuse (ensemblePruning)
            reuse_pruning = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="regression")
            pruning_predict_y = reuse_pruning.predict(user_data=test_x)
            pruning_score = loss_func_rmse(pruning_predict_y, test_y)
            pruning_score_list.append(pruning_score)
            logger.info(f"mixture reuse rmse (ensemble Pruning): {pruning_score}\n")

            learnware_rmse_list.append(rmse_list)

        single_list = np.array(learnware_rmse_list)
        avg_score_list = [np.mean(lst, axis=0) for lst in single_list]
        oracle_score_list = [np.min(lst, axis=0) for lst in single_list]

        logger.info(
            "RMSE of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f, Oracle performace: %.3f +/- %.3f"
            % (
                np.mean(single_score_list),
                np.std(single_score_list),
                np.mean(avg_score_list),
                np.std(avg_score_list),
                np.mean(oracle_score_list),
                np.std(oracle_score_list),
            )
        )
        logger.info(
            "Average Job Selector Reuse Performance: %.3f +/- %.3f"
            % (np.mean(job_selector_score_list), np.std(job_selector_score_list))
        )
        logger.info(
            "Averaging Ensemble Reuse Performance: %.3f +/- %.3f"
            % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
        )
        logger.info(
            "Selective Ensemble Reuse Performance: %.3f +/- %.3f"
            % (np.mean(pruning_score_list), np.std(pruning_score_list))
        )

    def labeled_homo_table_example(self):
        logger.info("Total Item: %d" % (len(self.market)))
        methods = ["user_model", "homo_single_aug", "homo_multiple_aug", "homo_multiple_avg", "homo_ensemble_pruning"]
        methods_to_retest = []
        recorders = {method: Recorder() for method in methods}

        user = self.benchmark.name
        for idx in range(self.benchmark.user_num):
            test_x, test_y = self.benchmark.get_test_data(user_ids=idx)
            test_x, test_y = test_x.values, test_y.values
            
            train_x, train_y = self.benchmark.get_train_data(user_ids=idx)
            train_x, train_y = train_x.values, train_y.values
            train_subsets = self.get_train_subsets(idx, train_x, train_y)

            user_stat_spec = generate_stat_spec(type="table", X=test_x)
            user_info = BaseUserInfo(
                semantic_spec=self.user_semantic, stat_info={"RKMETableSpecification": user_stat_spec}
            )
            logger.info(f"Searching Market for user: {user}_{idx}")

            search_result = self.market.search_learnware(user_info)
            single_result = search_result.get_single_results()
            multiple_result = search_result.get_multiple_results()

            logger.info(f"search result of user {user}_{idx}:")
            logger.info(
                f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
            )

            if len(multiple_result) > 0:
                mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
                logger.info(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
                mixture_learnware_list = multiple_result[0].learnwares
            else:
                mixture_learnware_list = [single_result[0].learnware]

            test_info = {"user": user, "idx": idx, "train_subsets": train_subsets, "test_x": test_x, "test_y": test_y}
            common_config = {"learnwares": mixture_learnware_list}
            method_configs = {
                "user_model": {"dataset": self.benchmark.name, "model_type": "lgb"},
                "homo_single_aug": {"single_learnware": [single_result[0].learnware]},
                "homo_multiple_aug": common_config,
                "homo_multiple_avg": common_config,
                "homo_ensemble_pruning": common_config
            }

            for method_name in methods:
                logger.info(f"Testing method {method_name}")
                test_info["method_name"] = method_name
                test_info["force"] = method_name in methods_to_retest
                test_info.update(method_configs[method_name])
                self.test_method(test_info, recorders, loss_func=loss_func_rmse)
        
        for method, recorder in recorders.items():
            recorder.save(os.path.join(self.curves_result_path, f"{user}_{method}_performance.json"))
        
        methods_to_plot = ["user_model", "homo_single_aug", "homo_ensemble_pruning"]
        plot_performance_curves(user, {method: recorders[method] for method in methods_to_plot}, task="Homo", n_labeled_list=n_labeled_list)