!657 add filter operation 'not_in' in lineagemgr

Merge pull request !657 from luopengting/lineage_not_in

mindinsight/lineagemgr/common/validator/model_parameter.py

@@ -74,7 +74,7 @@ class SearchModelConditionParameter(Schema):
     def check_dict_value_type(data, value_type):
         """Check dict value type and int scope."""
         for key, value in data.items():
-            if key == "in":
+            if key in ["in", "not_in"]:
                 if not isinstance(value, (list, tuple)):
                     raise ValidationError("The value of `in` operation must be list or tuple.")
             else:
@@ -101,10 +101,8 @@ class SearchModelConditionParameter(Schema):
     @staticmethod
     def check_operation(data):
         """Check input param's compare operation."""
-        if not set(data.keys()).issubset(['in', 'eq']):
-            raise ValidationError("Its operation should be `in` or `eq`.")
-        if len(data.keys()) > 1:
-            raise ValidationError("More than one operation.")
+        if not set(data.keys()).issubset(['in', 'eq', 'not_in']):
+            raise ValidationError("Its operation should be `eq`, `in` or `not_in`.")
 
     @validates("loss")
     def check_loss(self, data):
@@ -222,9 +220,9 @@ class SearchModelConditionParameter(Schema):
                                             .format(attr))
 
             for key in condition.keys():
-                if key not in ["eq", "lt", "gt", "le", "ge", "in"]:
+                if key not in ["eq", "lt", "gt", "le", "ge", "in", "not_in"]:
                     raise LineageParamValueError("The compare condition should be in "
-                                                 "('eq', 'lt', 'gt', 'le', 'ge', 'in').")
+                                                 "('eq', 'lt', 'gt', 'le', 'ge', 'in', 'not_in').")
 
             if attr.startswith('metric/'):
                 if len(attr) == 7:

mindinsight/lineagemgr/model.py

@@ -167,18 +167,19 @@ def _convert_relative_path_to_abspath(summary_base_dir, search_condition):
 
     summary_dir_condition = search_condition.get("summary_dir")
 
-    if 'in' in summary_dir_condition:
-        summary_paths = []
-        for summary_dir in summary_dir_condition.get('in'):
-            if summary_dir.startswith('./'):
-                abs_dir = os.path.join(
-                    summary_base_dir, summary_dir[2:]
-                )
-                abs_dir = validate_path(abs_dir)
-            else:
-                abs_dir = validate_path(summary_dir)
-            summary_paths.append(abs_dir)
-        search_condition.get('summary_dir')['in'] = summary_paths
+    for key in ['in', 'not_in']:
+        if key in summary_dir_condition:
+            summary_paths = []
+            for summary_dir in summary_dir_condition.get(key):
+                if summary_dir.startswith('./'):
+                    abs_dir = os.path.join(
+                        summary_base_dir, summary_dir[2:]
+                    )
+                    abs_dir = validate_path(abs_dir)
+                else:
+                    abs_dir = validate_path(summary_dir)
+                summary_paths.append(abs_dir)
+            search_condition.get('summary_dir')[key] = summary_paths
 
     if 'eq' in summary_dir_condition:
         summary_dir = summary_dir_condition.get('eq')

mindinsight/lineagemgr/querier/querier.py

@@ -66,6 +66,7 @@ class ExpressionType(enum.Enum):
     LE = 'le'
     GE = 'ge'
     IN = 'in'
+    NOT_IN = 'not_in'
 
     @classmethod
     def is_valid_exp(cls, key):
@@ -103,6 +104,8 @@ class ExpressionType(enum.Enum):
         try:
             if except_key == cls.IN.value:
                 state = operator.contains(except_value, actual_value)
+            elif except_key == cls.NOT_IN.value:
+                state = not operator.contains(except_value, actual_value)
             else:
                 state = getattr(operator, except_key)(actual_value, except_value)
         except TypeError:

tests/st/func/lineagemgr/test_model.py

@@ -762,23 +762,7 @@ class TestModelApi(TestCase):
             }
             self.assertRaisesRegex(
                 LineageSearchConditionParamError,
-                f'The parameter {condition_key} is invalid. Its operation should be `in` or `eq`.',
-                filter_summary_lineage,
-                None,
-                BASE_SUMMARY_DIR,
-                search_condition
-            )
-
-            # more than one operation in summary_dir and lineage_type
-            search_condition = {
-                condition_key: {
-                    'in': ['/xxx', '/yyy'],
-                    'eq': '/zzz',
-                }
-            }
-            self.assertRaisesRegex(
-                LineageSearchConditionParamError,
-                f'The parameter {condition_key} is invalid. More than one operation.',
+                f'The parameter {condition_key} is invalid. Its operation should be `eq`, `in` or `not_in`.',
                 filter_summary_lineage,
                 None,
                 BASE_SUMMARY_DIR,

tests/ut/lineagemgr/common/validator/test_validate.py

@@ -136,7 +136,7 @@ class TestValidateSearchModelCondition(TestCase):
         }
         self._assert_raise_of_mindinsight_exception(
             "The parameter loss_function is invalid. "
-            "Its operation should be `in` or `eq`.",
+            "Its operation should be `eq`, `in` or `not_in`.",
             condition
         )
 
@@ -160,7 +160,7 @@ class TestValidateSearchModelCondition(TestCase):
         }
         self._assert_raise_of_mindinsight_exception(
             "The parameter network is invalid. "
-            "Its operation should be `in` or `eq`.",
+            "Its operation should be `eq`, `in` or `not_in`.",
             condition
         )