From 2f06c14ae4661a679a8822355493e4fb2961ffaf Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 3 Dec 2023 14:06:12 +0800 Subject: [PATCH 1/9] [FIX] fix hetero import torch --- learnware/client/package_utils.py | 2 +- learnware/market/heterogeneous/__init__.py | 14 ++++++++++++-- learnware/market/heterogeneous/searcher.py | 6 ++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index f3266ae..dbc8679 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -30,7 +30,7 @@ def parse_pip_requirement(line: str): line = line.strip() if len(line) == 0 or line[0] in ("#", "-"): - return None + return None, None package_name, package_version = line, line for split_ch in ("=", ">", "<", "!", "~", " ", "="): diff --git a/learnware/market/heterogeneous/__init__.py b/learnware/market/heterogeneous/__init__.py index e7ab916..df0ad39 100644 --- a/learnware/market/heterogeneous/__init__.py +++ b/learnware/market/heterogeneous/__init__.py @@ -1,2 +1,12 @@ -from .organizer import HeteroMapTableOrganizer -from .searcher import HeteroSearcher +from ...utils import is_torch_available +from ...logger import get_module_logger + +logger = get_module_logger("market_hetero") + +if not is_torch_available(verbose=False): + HeteroMapTableOrganizer = None + HeteroSearcher = None + logger.error("HeteroMapTableOrganizer and HeteroSearcher are not available because 'torch' is not installed!") +else: + from .organizer import HeteroMapTableOrganizer + from .searcher import HeteroSearcher diff --git a/learnware/market/heterogeneous/searcher.py b/learnware/market/heterogeneous/searcher.py index f261703..7194f9c 100644 --- a/learnware/market/heterogeneous/searcher.py +++ b/learnware/market/heterogeneous/searcher.py @@ -1,11 +1,9 @@ -import traceback -from typing import Tuple, List +from typing import Optional from .utils import is_hetero from ..base import BaseUserInfo, SearchResults from ..easy import EasySearcher from ..utils import parse_specification_type -from ...learnware import Learnware from ...logger import get_module_logger @@ -14,7 +12,7 @@ logger = get_module_logger("hetero_searcher") class HeteroSearcher(EasySearcher): def __call__( - self, user_info: BaseUserInfo, check_status: int = None, max_search_num: int = 5, search_method: str = "greedy" + self, user_info: BaseUserInfo, check_status: Optional[int] = None, max_search_num: int = 5, search_method: str = "greedy" ) -> SearchResults: """Search learnwares based on user_info from learnwares with check_status. Employs heterogeneous learnware search if specific requirements are met, otherwise resorts to homogeneous search methods. From c28fb14cf142e45ef994df87f75f2f7448098fde Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 3 Dec 2023 14:07:19 +0800 Subject: [PATCH 2/9] [MNT] publish 0.2.0.6 beta version --- learnware/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/__init__.py b/learnware/__init__.py index eed5dde..b7fb924 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.0.5" +__version__ = "0.2.0.6" import os import json From f55a94e4442c9d475393de8e8d2681c2248a3d2e Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 3 Dec 2023 15:11:18 +0800 Subject: [PATCH 3/9] [FIX] fix conda deps filter --- learnware/client/package_utils.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index dbc8679..267c090 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -21,7 +21,7 @@ def try_to_run(args, timeout=10, retry=3): return result.stdout.decode() except subprocess.TimeoutExpired as e: pass - + raise subprocess.TimeoutExpired(args, timeout) @@ -37,11 +37,11 @@ def parse_pip_requirement(line: str): split_ch_index = package_name.find(split_ch) if split_ch_index != -1: package_name = package_name[:split_ch_index] - + split_ch_index = package_version.find(split_ch) if split_ch_index != -1: - package_version = package_version[split_ch_index + 1:] - + package_version = package_version[split_ch_index + 1 :] + if package_version == package_name: package_version = "" @@ -71,6 +71,7 @@ def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]: exist_packages: list of exist packages nonexist_packages: list of non-exist packages """ + def _filter_nonexist_pip_package_worker(package): # Return filtered package try: @@ -83,13 +84,13 @@ def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]: return package except Exception as e: logger.error(e) - + return None - + exist_packages = [] nonexist_packages = [] packages = [package for package in packages if package is not None] - + with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 5, 1)) as executor: results = executor.map(_filter_nonexist_pip_package_worker, packages) @@ -98,7 +99,7 @@ def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]: exist_packages.append(result) else: nonexist_packages.append(package) - + return exist_packages, nonexist_packages @@ -129,7 +130,14 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] last_bracket = stdout.rfind("\n{") if last_bracket != -1: stdout = stdout[last_bracket:] - return json.loads(stdout).get("bad_deps", []) + + stdout_json = json.loads(stdout) + if "error" in stdout_json: + if "bad_deps" in stdout_json: + return stdout_json["bad_deps"] + elif "packages" in stdout_json: + return stdout_json["packages"] + return [] org_yaml = { "channels": ["defaults"], From 9756792cacebac910fba1cffde44aeed8241dd00 Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 3 Dec 2023 15:22:58 +0800 Subject: [PATCH 4/9] [MNT] publish 0.2.0.7 beta version --- learnware/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/__init__.py b/learnware/__init__.py index b7fb924..6d86b3d 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.2.0.6" +__version__ = "0.2.0.7" import os import json From 72214b8116e84b8c725b4b52a113dc3e26baf8e5 Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 4 Dec 2023 13:26:06 +0800 Subject: [PATCH 5/9] [MNT] add type check in spec load --- learnware/specification/regular/image/rkme.py | 28 +++++++++++++------ learnware/specification/regular/table/rkme.py | 21 +++++++++----- .../specification/system/hetero_table.py | 14 +++++++--- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 81b68f7..821924b 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -1,7 +1,6 @@ from __future__ import annotations import codecs -import copy import functools import json import os @@ -17,8 +16,11 @@ from tqdm import tqdm from . import cnn_gp from ..base import RegularStatSpecification from ..table.rkme import rkme_solve_qp +from ....logger import get_module_logger from ....utils import choose_device, allocate_cuda_idx +logger = get_module_logger("image_rkme") + class RKMEImageSpecification(RegularStatSpecification): # INNER_PRODUCT_COUNT = 0 @@ -127,8 +129,10 @@ class RKMEImageSpecification(RegularStatSpecification): try: from torchvision.transforms import Resize except ModuleNotFoundError: - raise ModuleNotFoundError(f"RKMEImageSpecification is not available because 'torchvision' is not installed! Please install it manually." ) - + raise ModuleNotFoundError( + f"RKMEImageSpecification is not available because 'torchvision' is not installed! Please install it manually." + ) + if X.shape[2] != RKMEImageSpecification.IMAGE_WIDTH or X.shape[3] != RKMEImageSpecification.IMAGE_WIDTH: X = Resize((RKMEImageSpecification.IMAGE_WIDTH, RKMEImageSpecification.IMAGE_WIDTH), antialias=True)(X) @@ -154,12 +158,14 @@ class RKMEImageSpecification(RegularStatSpecification): with torch.no_grad(): x_features = self._generate_random_feature(X_train, random_models=random_models) self._update_beta(x_features, nonnegative_beta, random_models=random_models) - + try: import torch_optimizer except ModuleNotFoundError: - raise ModuleNotFoundError(f"RKMEImageSpecification is not available because 'torch-optimizer' is not installed! Please install it manually.") - + raise ModuleNotFoundError( + f"RKMEImageSpecification is not available because 'torch-optimizer' is not installed! Please install it manually." + ) + optimizer = torch_optimizer.AdaBelief([{"params": [self.z]}], lr=step_size, eps=1e-16) for _ in tqdm(range(steps)) if verbose else range(steps): @@ -385,9 +391,13 @@ class RKMEImageSpecification(RegularStatSpecification): self.beta = self.beta.to(self._device) self.z = self.z.to(self._device) - return True - else: - return False + if self.type == self.__class__.__name__: + logger.error( + f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" + ) + return True + + return False def _get_zca_matrix(X, reg_coef=0.1): diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index f335f4d..af8ff20 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -140,15 +140,17 @@ class RKMETableSpecification(RegularStatSpecification): if isinstance(X, np.ndarray): X = X.astype("float32") X = torch.from_numpy(X) - + X = X.to(self._device) - + try: from fast_pytorch_kmeans import KMeans except ModuleNotFoundError: - raise ModuleNotFoundError(f"RKMETableSpecification is not available because 'fast_pytorch_kmeans' is not installed! Please install it manually." ) + raise ModuleNotFoundError( + f"RKMETableSpecification is not available because 'fast_pytorch_kmeans' is not installed! Please install it manually." + ) - kmeans = KMeans(n_clusters=K, mode='euclidean', max_iter=100, verbose=0) + kmeans = KMeans(n_clusters=K, mode="euclidean", max_iter=100, verbose=0) kmeans.fit(X) self.z = kmeans.centroids.double() @@ -455,9 +457,14 @@ class RKMETableSpecification(RegularStatSpecification): for d in self.get_states(): if d in rkme_load.keys(): setattr(self, d, rkme_load[d]) - return True - else: - return False + + if self.type == self.__class__.__name__: + logger.error( + f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" + ) + return True + + return False class RKMEStatSpecification(RKMETableSpecification): diff --git a/learnware/specification/system/hetero_table.py b/learnware/specification/system/hetero_table.py index 3987040..ad9d7a8 100644 --- a/learnware/specification/system/hetero_table.py +++ b/learnware/specification/system/hetero_table.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import copy import json import torch import codecs @@ -10,8 +9,11 @@ import numpy as np from .base import SystemStatSpecification from ..regular import RKMETableSpecification from ..regular.table.rkme import torch_rbf_kernel +from ...logger import get_module_logger from ...utils import choose_device, allocate_cuda_idx +logger = get_module_logger("hetero_map_table_spec") + class HeteroMapTableSpecification(SystemStatSpecification): """Heterogeneous Map-Table Specification""" @@ -135,9 +137,13 @@ class HeteroMapTableSpecification(SystemStatSpecification): if d in embedding_load.keys(): setattr(self, d, embedding_load[d]) - return True - else: - return False + if self.type == self.__class__.__name__: + logger.error( + f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" + ) + return True + + return False def save(self, filepath: str) -> bool: """Save the computed HeteroMapTableSpecification to a specified path in JSON format. From a7b8f8d12fa7e3bf27525b39b67d3b4bb8e8e5e6 Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 4 Dec 2023 13:36:16 +0800 Subject: [PATCH 6/9] [FIX] add spec.dist() check --- learnware/market/easy/checker.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/learnware/market/easy/checker.py b/learnware/market/easy/checker.py index 7995d94..57cae22 100644 --- a/learnware/market/easy/checker.py +++ b/learnware/market/easy/checker.py @@ -95,8 +95,8 @@ class EasyStatChecker(BaseChecker): logger.warning(f"The learnware [{learnware.id}] is instantiated failed! Due to {e}.") return self.INVALID_LEARNWARE, traceback.format_exc() try: - learnware_model = learnware.get_model() # Check input shape + learnware_model = learnware.get_model() input_shape = learnware_model.input_shape if semantic_spec["Data"]["Values"][0] == "Table" and input_shape != ( @@ -106,14 +106,18 @@ class EasyStatChecker(BaseChecker): logger.warning(message) return self.INVALID_LEARNWARE, message + # Check statistical specification spec_type = parse_specification_type(learnware.get_specification().stat_spec) if spec_type is None: message = f"No valid specification is found in stat spec {spec_type}" logger.warning(message) return self.INVALID_LEARNWARE, message + # Check if statistical specification is computable in dist() + stat_spec = learnware.get_specification().get_stat_spec_by_name(spec_type) + stat_spec.dist(stat_spec) + if spec_type == "RKMETableSpecification": - stat_spec = learnware.get_specification().get_stat_spec_by_name(spec_type) if not isinstance(input_shape, tuple) or not all(isinstance(item, int) for item in input_shape): raise ValueError( f"For RKMETableSpecification, input_shape should be tuple of int, but got {input_shape}" @@ -124,14 +128,17 @@ class EasyStatChecker(BaseChecker): logger.warning(message) return self.INVALID_LEARNWARE, message inputs = np.random.randn(10, *input_shape) + elif spec_type == "RKMETextSpecification": inputs = EasyStatChecker._generate_random_text_list(10) + elif spec_type == "RKMEImageSpecification": if not isinstance(input_shape, tuple) or not all(isinstance(item, int) for item in input_shape): raise ValueError( f"For RKMEImageSpecification, input_shape should be tuple of int, but got {input_shape}" ) inputs = np.random.randint(0, 255, size=(10, *input_shape)) + else: raise ValueError(f"not supported spec type for spec_type = {spec_type}") From d9a0ccc9588e771f1f0a7799ec688fe5e31458ec Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 4 Dec 2023 13:46:47 +0800 Subject: [PATCH 7/9] [FIX] fix details --- learnware/specification/regular/image/rkme.py | 2 +- learnware/specification/regular/table/rkme.py | 2 +- learnware/specification/system/hetero_table.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 821924b..448ae62 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -391,7 +391,7 @@ class RKMEImageSpecification(RegularStatSpecification): self.beta = self.beta.to(self._device) self.z = self.z.to(self._device) - if self.type == self.__class__.__name__: + if self.type != self.__class__.__name__: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index af8ff20..78e4996 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -458,7 +458,7 @@ class RKMETableSpecification(RegularStatSpecification): if d in rkme_load.keys(): setattr(self, d, rkme_load[d]) - if self.type == self.__class__.__name__: + if self.type != self.__class__.__name__: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) diff --git a/learnware/specification/system/hetero_table.py b/learnware/specification/system/hetero_table.py index ad9d7a8..5bec254 100644 --- a/learnware/specification/system/hetero_table.py +++ b/learnware/specification/system/hetero_table.py @@ -137,7 +137,7 @@ class HeteroMapTableSpecification(SystemStatSpecification): if d in embedding_load.keys(): setattr(self, d, embedding_load[d]) - if self.type == self.__class__.__name__: + if self.type != self.__class__.__name__: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) From 61ed6da6c43cadd5624e1249f5d94c4a11d94ace Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 4 Dec 2023 13:57:49 +0800 Subject: [PATCH 8/9] [FIX] fix bugs --- learnware/specification/regular/image/rkme.py | 5 +++-- learnware/specification/regular/table/rkme.py | 7 ++++--- learnware/specification/system/hetero_table.py | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 448ae62..32809b7 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -391,11 +391,12 @@ class RKMEImageSpecification(RegularStatSpecification): self.beta = self.beta.to(self._device) self.z = self.z.to(self._device) - if self.type != self.__class__.__name__: + if self.type == self.__class__.__name__: + return True + else: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) - return True return False diff --git a/learnware/specification/regular/table/rkme.py b/learnware/specification/regular/table/rkme.py index 78e4996..046a956 100644 --- a/learnware/specification/regular/table/rkme.py +++ b/learnware/specification/regular/table/rkme.py @@ -6,7 +6,7 @@ import json import codecs import scipy import numpy as np -from qpsolvers import solve_qp, Problem, solve_problem +from qpsolvers import Problem, solve_problem from collections import Counter from typing import Any, Union @@ -458,11 +458,12 @@ class RKMETableSpecification(RegularStatSpecification): if d in rkme_load.keys(): setattr(self, d, rkme_load[d]) - if self.type != self.__class__.__name__: + if self.type == self.__class__.__name__: + return True + else: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) - return True return False diff --git a/learnware/specification/system/hetero_table.py b/learnware/specification/system/hetero_table.py index 5bec254..2a8ab6b 100644 --- a/learnware/specification/system/hetero_table.py +++ b/learnware/specification/system/hetero_table.py @@ -137,11 +137,12 @@ class HeteroMapTableSpecification(SystemStatSpecification): if d in embedding_load.keys(): setattr(self, d, embedding_load[d]) - if self.type != self.__class__.__name__: + if self.type == self.__class__.__name__: + return True + else: logger.error( f"The type of loaded RKME ({self.type}) is different from the expected type ({self.__class__.__name__})!" ) - return True return False From 13bb47359ddb453616dc9921ba4b28cb115fd0ac Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 4 Dec 2023 14:10:18 +0800 Subject: [PATCH 9/9] [FIX] fix bugs in test_hetero --- tests/test_hetero_market/test_hetero.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_hetero_market/test_hetero.py b/tests/test_hetero_market/test_hetero.py index 41b4261..7b0740b 100644 --- a/tests/test_hetero_market/test_hetero.py +++ b/tests/test_hetero_market/test_hetero.py @@ -5,10 +5,10 @@ import copy import joblib import zipfile import numpy as np +import multiprocessing from sklearn.linear_model import Ridge from sklearn.datasets import make_regression from shutil import copyfile, rmtree -from multiprocessing import Pool from learnware.client import LearnwareClient from sklearn.metrics import mean_squared_error @@ -121,7 +121,8 @@ class TestMarket(unittest.TestCase): dir_path = os.path.join(curr_root, "learnware_pool") # Execute multi-process checking using Pool - with Pool() as pool: + mp_context = multiprocessing.get_context("spawn") + with mp_context.Pool() as pool: results = pool.starmap(check_learnware, [(name, dir_path) for name in os.listdir(dir_path)]) # Use an assert statement to ensure that all checks return True