Browse Source

add mixed type for customized, set operation as None when it is empty or None

tags/v0.2.0-alpha
luopengting 5 years ago
parent
commit
c97736438a
4 changed files with 49 additions and 21 deletions
  1. +38
    -15
      mindinsight/lineagemgr/querier/querier.py
  2. +2
    -3
      mindinsight/lineagemgr/summary/_summary_adapter.py
  3. +6
    -0
      tests/ut/lineagemgr/querier/event_data.py
  4. +3
    -3
      tests/ut/lineagemgr/querier/test_querier.py

+ 38
- 15
mindinsight/lineagemgr/querier/querier.py View File

@@ -288,8 +288,8 @@ class Querier:
try:
cmp_result = (value1 > value2) - (value1 < value2)
except TypeError:
type1 = str(type(value1))
type2 = str(type(value2))
type1 = type(value1).__name__
type2 = type(value2).__name__
cmp_result = (type1 > type2) - (type1 < type2)
return cmp_result

@@ -314,19 +314,7 @@ class Querier:

offset_results = self._handle_limit_and_offset(condition, results)

customized = dict()
for offset_result in offset_results:
for obj_name in ["metric", "user_defined"]:
obj = getattr(offset_result, obj_name)
require = bool(obj_name == "metric")
if obj and isinstance(obj, dict):
for key, value in obj.items():
label = f'{obj_name}/{key}'
customized[label] = dict()
customized[label]["label"] = label
# user defined info is not displayed by default
customized[label]["required"] = require
customized[label]["type"] = type(value).__name__
customized = self._organize_customized(offset_results)

lineage_types = condition.get(ConditionParam.LINEAGE_TYPE.value)
lineage_types = self._get_lineage_types(lineage_types)
@@ -348,6 +336,41 @@ class Querier:

return lineage_info

def _organize_customized(self, offset_results):
"""Organize customized."""
customized = dict()
for offset_result in offset_results:
for obj_name in ["metric", "user_defined"]:
self._organize_customized_item(customized, offset_result, obj_name)

# If types contain numbers and string, it will be "mixed".
# If types contain "int" and "float", it will be "float".
for key, value in customized.items():
types = value["type"]
if len(types) == 1:
customized[key]["type"] = list(types)[0]
elif types.issubset(["int", "float"]):
customized[key]["type"] = "float"
else:
customized[key]["type"] = "mixed"
return customized

def _organize_customized_item(self, customized, offset_result, obj_name):
"""Organize customized item."""
obj = getattr(offset_result, obj_name)
require = bool(obj_name == "metric")
if obj and isinstance(obj, dict):
for key, value in obj.items():
label = f'{obj_name}/{key}'
current_type = type(value).__name__
if customized.get(label) is None:
customized[label] = dict()
customized[label]["label"] = label
# user defined info is not displayed by default
customized[label]["required"] = require
customized[label]["type"] = set()
customized[label]["type"].add(current_type)

def _get_lineage_types(self, lineage_type_param):
"""
Get lineage types.


+ 2
- 3
mindinsight/lineagemgr/summary/_summary_adapter.py View File

@@ -70,13 +70,13 @@ def _package_current_dataset(operation, message):
message (Operation): Operation proto message.
"""
for key, value in operation.items():
if key == "operations":
if value and key == "operations":
for operator in value:
_package_enhancement_operation(
operator,
message.operations.add()
)
elif key == "sampler":
elif value and key == "sampler":
_package_enhancement_operation(
value,
message.sampler
@@ -93,7 +93,6 @@ def _package_enhancement_operation(operation, message):
operation (dict): Enhancement operation.
message (Operation): Enhancement operation proto message.
"""

for key, value in operation.items():
if isinstance(value, list):
if all(isinstance(ele, int) for ele in value):


+ 6
- 0
tests/ut/lineagemgr/querier/event_data.py View File

@@ -204,6 +204,12 @@ CUSTOMIZED_1 = {
'metric/mse': {'label': 'metric/mse', 'required': True, 'type': 'float'}
}

CUSTOMIZED_2 = {
'metric/accuracy': {'label': 'metric/accuracy', 'required': True, 'type': 'mixed'},
'metric/mae': {'label': 'metric/mae', 'required': True, 'type': 'float'},
'metric/mse': {'label': 'metric/mse', 'required': True, 'type': 'float'}
}

METRIC_1 = {
'accuracy': 1.0000002,
'mae': 2.00000002,


+ 3
- 3
tests/ut/lineagemgr/querier/test_querier.py View File

@@ -463,7 +463,7 @@ class TestQuerier(TestCase):
def test_filter_summary_lineage_success_4(self):
"""Test the success of filter_summary_lineage."""
expected_result = {
'customized': event_data.CUSTOMIZED_0,
'customized': event_data.CUSTOMIZED_2,
'object': [
LINEAGE_FILTRATION_0,
LINEAGE_FILTRATION_1,
@@ -500,7 +500,7 @@ class TestQuerier(TestCase):
'sorted_type': 'ascending'
}
expected_result = {
'customized': event_data.CUSTOMIZED_0,
'customized': event_data.CUSTOMIZED_2,
'object': [
LINEAGE_FILTRATION_0,
LINEAGE_FILTRATION_5,
@@ -522,7 +522,7 @@ class TestQuerier(TestCase):
'sorted_type': 'descending'
}
expected_result = {
'customized': event_data.CUSTOMIZED_1,
'customized': event_data.CUSTOMIZED_2,
'object': [
LINEAGE_FILTRATION_6,
LINEAGE_FILTRATION_4,


Loading…
Cancel
Save