Huawei_Technology
/
mindspore-mindinsight
Not watched
1
0
Fork 0
Code
Releases 11 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

fix ut/st about cmetrics

tags/v0.2.0-alpha
chenchao99 5 years ago
parent
228a448b46
commit
4a614930d0
8 changed files with 472 additions and 915 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +134
    -208
      tests/st/func/lineagemgr/api/test_model_api.py
  2. +3
    -46
      tests/st/func/lineagemgr/conftest.py
  3. +40
    -60
      tests/ut/backend/lineagemgr/test_lineage_api.py
  4. +20
    -40
      tests/ut/lineagemgr/collection/model/test_model_lineage.py
  5. +35
    -58
      tests/ut/lineagemgr/common/test_path_parser.py
  6. +107
    -101
      tests/ut/lineagemgr/common/validator/test_validate.py
  7. +4
    -46
      tests/ut/lineagemgr/querier/event_data.py
  8. +129
    -356
      tests/ut/lineagemgr/querier/test_querier.py

+ 134
- 208
tests/st/func/lineagemgr/api/test_model_api.py View File

@@ -33,6 +33,96 @@ from mindinsight.lineagemgr.common.exceptions.exceptions import \
from ..conftest import BASE_SUMMARY_DIR, SUMMARY_DIR, SUMMARY_DIR_2, DATASET_GRAPH


LINEAGE_INFO_RUN1 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'metric': {
'accuracy': 0.78
},
'hyper_parameters': {
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'epoch': 14,
'parallel_mode': 'stand_alone',
'device_num': 2,
'batch_size': 32
},
'algorithm': {
'network': 'ResNet'
},
'train_dataset': {
'train_dataset_size': 731
},
'valid_dataset': {
'valid_dataset_size': 10240
},
'model': {
'path': '{"ckpt": "'
+ BASE_SUMMARY_DIR + '/run1/CKPtest_model.ckpt"}',
'size': 64
},
'dataset_graph': DATASET_GRAPH
}
LINEAGE_FILTRATION_EXCEPT_RUN = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 1024,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 10,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 64,
'metric': {},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
LINEAGE_FILTRATION_RUN1 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
LINEAGE_FILTRATION_RUN2 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
}


@pytest.mark.usefixtures("create_summary_dir")
class TestModelApi(TestCase):
"""Test lineage information query interface."""
@@ -67,36 +157,7 @@ class TestModelApi(TestCase):
total_res = get_summary_lineage(SUMMARY_DIR)
partial_res1 = get_summary_lineage(SUMMARY_DIR, ['hyper_parameters'])
partial_res2 = get_summary_lineage(SUMMARY_DIR, ['metric', 'algorithm'])
expect_total_res = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'metric': {
'accuracy': 0.78
},
'hyper_parameters': {
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'epoch': 14,
'parallel_mode': 'stand_alone',
'device_num': 2,
'batch_size': 32
},
'algorithm': {
'network': 'ResNet'
},
'train_dataset': {
'train_dataset_size': 731
},
'valid_dataset': {
'valid_dataset_size': 10240
},
'model': {
'path': '{"ckpt": "'
+ BASE_SUMMARY_DIR + '/run1/CKPtest_model.ckpt"}',
'size': 64
},
'dataset_graph': DATASET_GRAPH
}
expect_total_res = LINEAGE_INFO_RUN1
expect_partial_res1 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'hyper_parameters': {
@@ -139,7 +200,7 @@ class TestModelApi(TestCase):
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_get_summary_lineage_exception(self):
def test_get_summary_lineage_exception_1(self):
"""Test the interface of get_summary_lineage with exception."""
# summary path does not exist
self.assertRaisesRegex(
@@ -183,6 +244,14 @@ class TestModelApi(TestCase):
keys=None
)

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_get_summary_lineage_exception_2(self):
"""Test the interface of get_summary_lineage with exception."""
# keys is invalid
self.assertRaisesRegex(
LineageParamValueError,
@@ -250,64 +319,9 @@ class TestModelApi(TestCase):
"""Test the interface of filter_summary_lineage."""
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 1024,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 10,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 64,
'metric': {},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
}
LINEAGE_FILTRATION_EXCEPT_RUN,
LINEAGE_FILTRATION_RUN1,
LINEAGE_FILTRATION_RUN2
],
'count': 3
}
@@ -357,46 +371,8 @@ class TestModelApi(TestCase):
}
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
LINEAGE_FILTRATION_RUN2,
LINEAGE_FILTRATION_RUN1
],
'count': 2
}
@@ -432,46 +408,8 @@ class TestModelApi(TestCase):
}
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
LINEAGE_FILTRATION_RUN2,
LINEAGE_FILTRATION_RUN1
],
'count': 2
}
@@ -498,44 +436,8 @@ class TestModelApi(TestCase):
}
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 1024,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 10,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 64,
'metric': {},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
LINEAGE_FILTRATION_EXCEPT_RUN,
LINEAGE_FILTRATION_RUN1
],
'count': 2
}
@@ -674,6 +576,14 @@ class TestModelApi(TestCase):
search_condition
)

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_filter_summary_lineage_exception_3(self):
"""Test the abnormal execution of the filter_summary_lineage interface."""
# the condition of offset is invalid
search_condition = {
'offset': 1.0
@@ -712,6 +622,14 @@ class TestModelApi(TestCase):
search_condition
)

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_filter_summary_lineage_exception_4(self):
"""Test the abnormal execution of the filter_summary_lineage interface."""
# the sorted_type not supported
search_condition = {
'sorted_name': 'summary_dir',
@@ -753,6 +671,14 @@ class TestModelApi(TestCase):
search_condition
)

@pytest.mark.level0
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_gpu_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_filter_summary_lineage_exception_5(self):
"""Test the abnormal execution of the filter_summary_lineage interface."""
# the summary dir is invalid in search condition
search_condition = {
'summary_dir': {
@@ -811,7 +737,7 @@ class TestModelApi(TestCase):
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_cpu
@pytest.mark.env_single
def test_filter_summary_lineage_exception_3(self):
def test_filter_summary_lineage_exception_6(self):
"""Test the abnormal execution of the filter_summary_lineage interface."""
# gt > lt
search_condition1 = {


+ 3
- 46
tests/st/func/lineagemgr/conftest.py View File

@@ -22,6 +22,8 @@ import tempfile
import pytest

from ....utils import mindspore
from ....utils.mindspore.dataset.engine.serializer_deserializer import \
SERIALIZED_PIPELINE

sys.modules['mindspore'] = mindspore

@@ -32,52 +34,7 @@ SUMMARY_DIR_3 = os.path.join(BASE_SUMMARY_DIR, 'except_run')

COLLECTION_MODULE = 'TestModelLineage'
API_MODULE = 'TestModelApi'
DATASET_GRAPH = {
'op_type': 'BatchDataset',
'op_module': 'minddata.dataengine.datasets',
'num_parallel_workers': None,
'drop_remainder': True,
'batch_size': 10,
'children': [
{
'op_type': 'MapDataset',
'op_module': 'minddata.dataengine.datasets',
'num_parallel_workers': None,
'input_columns': [
'label'
],
'output_columns': [
None
],
'operations': [
{
'tensor_op_module': 'minddata.transforms.c_transforms',
'tensor_op_name': 'OneHot',
'num_classes': 10
}
],
'children': [
{
'op_type': 'MnistDataset',
'shard_id': None,
'num_shards': None,
'op_module': 'minddata.dataengine.datasets',
'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData',
'num_parallel_workers': None,
'shuffle': None,
'num_samples': 100,
'sampler': {
'sampler_module': 'minddata.dataengine.samplers',
'sampler_name': 'RandomSampler',
'replacement': True,
'num_samples': 100
},
'children': []
}
]
}
]
}
DATASET_GRAPH = SERIALIZED_PIPELINE

def get_module_name(nodeid):
"""Get the module name from nodeid."""


+ 40
- 60
tests/ut/backend/lineagemgr/test_lineage_api.py View File

@@ -24,6 +24,42 @@ from mindinsight.lineagemgr.common.exceptions.exceptions import \
LineageQuerySummaryDataError


LINEAGE_FILTRATION_BASE = {
'accuracy': None,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 12,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
}
LINEAGE_FILTRATION_RUN1 = {
'accuracy': 0.78,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': 64,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
}


class TestSearchModel(TestCase):
"""Test the restful api of search_model."""

@@ -42,39 +78,11 @@ class TestSearchModel(TestCase):
'object': [
{
'summary_dir': base_dir,
'accuracy': None,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 12,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
**LINEAGE_FILTRATION_BASE
},
{
'summary_dir': os.path.join(base_dir, 'run1'),
'accuracy': 0.78,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': 64,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
**LINEAGE_FILTRATION_RUN1
}
],
'count': 2
@@ -93,39 +101,11 @@ class TestSearchModel(TestCase):
'object': [
{
'summary_dir': './',
'accuracy': None,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 12,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
**LINEAGE_FILTRATION_BASE
},
{
'summary_dir': './run1',
'accuracy': 0.78,
'mae': None,
'mse': None,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 64,
'test_dataset_path': None,
'test_dataset_count': 64,
'network': 'str',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 128
**LINEAGE_FILTRATION_RUN1
}
],
'count': 2


+ 20
- 40
tests/ut/lineagemgr/collection/model/test_model_lineage.py View File

@@ -62,14 +62,10 @@ class TestModelLineage(TestCase):
self.assertTrue(f'Invalid value for raise_exception.' in str(context.exception))

@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.ds')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'LineageSummary.record_dataset_graph')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'validate_summary_record')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'AnalyzeObject.get_optimizer_by_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_dataset_graph')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_optimizer_by_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network')
def test_begin(self, *args):
"""Test TrainLineage.begin method."""
@@ -84,14 +80,10 @@ class TestModelLineage(TestCase):
args[4].assert_called()

@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.ds')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'LineageSummary.record_dataset_graph')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'validate_summary_record')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'AnalyzeObject.get_optimizer_by_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.'
'AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_dataset_graph')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_optimizer_by_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network')
def test_begin_error(self, *args):
"""Test TrainLineage.begin method."""
@@ -124,15 +116,11 @@ class TestModelLineage(TestCase):
train_lineage.begin(self.my_run_context(run_context))
self.assertTrue('The parameter optimizer is invalid.' in str(context.exception))

@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context')
@mock.patch('builtins.float')
@@ -160,15 +148,11 @@ class TestModelLineage(TestCase):
train_lineage.end(self.run_context)
self.assertTrue('Invalid TrainLineage run_context.' in str(context.exception))

@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context')
@mock.patch('builtins.float')
@@ -188,15 +172,11 @@ class TestModelLineage(TestCase):
train_lineage.end(self.my_run_context(self.run_context))
self.assertTrue('End error in TrainLineage:' in str(context.exception))

@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch(
'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network')
@mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context')
@mock.patch('builtins.float')


+ 35
- 58
tests/ut/lineagemgr/common/test_path_parser.py View File

@@ -20,23 +20,44 @@ from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
from mindinsight.lineagemgr.common.path_parser import SummaryPathParser


MOCK_SUMMARY_DIRS = [
{
'relative_path': './relative_path0'
},
{
'relative_path': './'
},
{
'relative_path': './relative_path1'
}
]
MOCK_SUMMARIES = [
{
'file_name': 'file0',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file0_lineage',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file1',
'create_time': datetime.fromtimestamp(1582031971)
},
{
'file_name': 'file1_lineage',
'create_time': datetime.fromtimestamp(1582031971)
}
]


class TestSummaryPathParser(TestCase):
"""Test the class of SummaryPathParser."""

@mock.patch.object(SummaryWatcher, 'list_summary_directories')
def test_get_summary_dirs(self, *args):
"""Test the function of get_summary_dirs."""
args[0].return_value = [
{
'relative_path': './relative_path0'
},
{
'relative_path': './'
},
{
'relative_path': './relative_path1'
}
]
args[0].return_value = MOCK_SUMMARY_DIRS

expected_result = [
'/path/to/base/relative_path0',
@@ -54,24 +75,7 @@ class TestSummaryPathParser(TestCase):
@mock.patch.object(SummaryWatcher, 'list_summaries')
def test_get_latest_lineage_summary(self, *args):
"""Test the function of get_latest_lineage_summary."""
args[0].return_value = [
{
'file_name': 'file0',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file0_lineage',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file1',
'create_time': datetime.fromtimestamp(1582031971)
},
{
'file_name': 'file1_lineage',
'create_time': datetime.fromtimestamp(1582031971)
}
]
args[0].return_value = MOCK_SUMMARIES
summary_dir = '/path/to/summary_dir'
result = SummaryPathParser.get_latest_lineage_summary(summary_dir)
self.assertEqual('/path/to/summary_dir/file1_lineage', result)
@@ -119,35 +123,8 @@ class TestSummaryPathParser(TestCase):
@mock.patch.object(SummaryWatcher, 'list_summary_directories')
def test_get_latest_lineage_summaries(self, *args):
"""Test the function of get_latest_lineage_summaries."""
args[0].return_value = [
{
'relative_path': './relative_path0'
},
{
'relative_path': './'
},
{
'relative_path': './relative_path1'
}
]
args[1].return_value = [
{
'file_name': 'file0',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file0_lineage',
'create_time': datetime.fromtimestamp(1582031970)
},
{
'file_name': 'file1',
'create_time': datetime.fromtimestamp(1582031971)
},
{
'file_name': 'file1_lineage',
'create_time': datetime.fromtimestamp(1582031971)
}
]
args[0].return_value = MOCK_SUMMARY_DIRS
args[1].return_value = MOCK_SUMMARIES

expected_result = [
'/path/to/base/relative_path0/file1_lineage',


+ 107
- 101
tests/ut/lineagemgr/common/validator/test_validate.py View File

@@ -26,27 +26,23 @@ from mindinsight.utils.exceptions import MindInsightException

class TestValidateSearchModelCondition(TestCase):
"""Test the mothod of validate_search_model_condition."""
def test_validate_search_model_condition(self):
"""Test the mothod of validate_search_model_condition."""
def test_validate_search_model_condition_param_type_error(self):
"""Test the mothod of validate_search_model_condition with LineageParamTypeError."""
condition = {
'summary_dir': 'xxx'
}
self.assertRaisesRegex(
LineageParamTypeError,
self._assert_raise_of_lineage_param_type_error(
'The search_condition element summary_dir should be dict.',
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

def test_validate_search_model_condition_param_value_error(self):
"""Test the mothod of validate_search_model_condition with LineageParamValueError."""
condition = {
'xxx': 'xxx'
}
self.assertRaisesRegex(
LineageParamValueError,
self._assert_raise_of_lineage_param_value_error(
'The search attribute not supported.',
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -55,22 +51,38 @@ class TestValidateSearchModelCondition(TestCase):
'xxx': 'xxx'
}
}
self.assertRaisesRegex(
LineageParamValueError,
self._assert_raise_of_lineage_param_value_error(
"The compare condition should be in",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

condition = {
1: {
"ge": 8.0
}
}
self._assert_raise_of_lineage_param_value_error(
"The search attribute not supported.",
condition
)

condition = {
'metric_': {
"ge": 8.0
}
}
self._assert_raise_of_lineage_param_value_error(
"The search attribute not supported.",
condition
)

def test_validate_search_model_condition_mindinsight_exception_1(self):
"""Test the mothod of validate_search_model_condition with MindinsightException."""
condition = {
"offset": 100001
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"Invalid input offset. 0 <= offset <= 100000",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -80,11 +92,9 @@ class TestValidateSearchModelCondition(TestCase):
},
'limit': 10
}
self.assertRaisesRegex(
MindInsightException,
"The parameter summary_dir is invalid. It should be a dict and the value should be a string",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter summary_dir is invalid. It should be a dict and "
"the value should be a string",
condition
)

@@ -93,11 +103,9 @@ class TestValidateSearchModelCondition(TestCase):
'in': 1.0
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter learning_rate is invalid. It should be a dict and "
"the value should be a float or a integer",
condition
)

@@ -106,24 +114,22 @@ class TestValidateSearchModelCondition(TestCase):
'lt': True
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter learning_rate is invalid. It should be a dict and "
"the value should be a float or a integer",
condition
)

def test_validate_search_model_condition_mindinsight_exception_2(self):
"""Test the mothod of validate_search_model_condition with MindinsightException."""
condition = {
'learning_rate': {
'gt': [1.0]
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter learning_rate is invalid. It should be a dict and "
"the value should be a float or a integer",
condition
)

@@ -132,11 +138,9 @@ class TestValidateSearchModelCondition(TestCase):
'ge': 1
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter loss_function is invalid. It should be a dict and the value should be a string",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter loss_function is invalid. It should be a dict and "
"the value should be a string",
condition
)

@@ -145,12 +149,9 @@ class TestValidateSearchModelCondition(TestCase):
'in': 2
}
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"The parameter train_dataset_count is invalid. It should be a dict "
"and the value should be a integer between 0",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -162,14 +163,14 @@ class TestValidateSearchModelCondition(TestCase):
'eq': 'xxx'
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter network is invalid. It should be a dict and the value should be a string",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter network is invalid. It should be a dict and "
"the value should be a string",
condition
)

def test_validate_search_model_condition_mindinsight_exception_3(self):
"""Test the mothod of validate_search_model_condition with MindinsightException."""
condition = {
'batch_size': {
'lt': 2,
@@ -179,11 +180,8 @@ class TestValidateSearchModelCondition(TestCase):
'eq': 222
}
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"The parameter batch_size is invalid. It should be a non-negative integer.",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -192,12 +190,9 @@ class TestValidateSearchModelCondition(TestCase):
'lt': -2
}
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"The parameter test_dataset_count is invalid. It should be a dict "
"and the value should be a integer between 0",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -206,11 +201,8 @@ class TestValidateSearchModelCondition(TestCase):
'lt': False
}
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"The parameter epoch is invalid. It should be a positive integer.",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

@@ -219,65 +211,79 @@ class TestValidateSearchModelCondition(TestCase):
"ge": ""
}
}
self.assertRaisesRegex(
MindInsightException,
"The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter learning_rate is invalid. It should be a dict and "
"the value should be a float or a integer",
condition
)

def test_validate_search_model_condition_mindinsight_exception_4(self):
"""Test the mothod of validate_search_model_condition with MindinsightException."""
condition = {
"train_dataset_count": {
"ge": 8.0
}
}
self.assertRaisesRegex(
MindInsightException,
self._assert_raise_of_mindinsight_exception(
"The parameter train_dataset_count is invalid. It should be a dict "
"and the value should be a integer between 0",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

condition = {
1: {
"ge": 8.0
'metric_attribute': {
'ge': 'xxx'
}
}
self.assertRaisesRegex(
LineageParamValueError,
"The search attribute not supported.",
validate_search_model_condition,
SearchModelConditionParameter,
self._assert_raise_of_mindinsight_exception(
"The parameter metric_attribute is invalid. "
"It should be a dict and the value should be a float or a integer",
condition
)

condition = {
'metric_': {
"ge": 8.0
}
}
LineageParamValueError('The search attribute not supported.')
self.assertRaisesRegex(
LineageParamValueError,
"The search attribute not supported.",
validate_search_model_condition,
SearchModelConditionParameter,
condition
)
def _assert_raise(self, exception, msg, condition):
"""
Assert raise by unittest.

condition = {
'metric_attribute': {
'ge': 'xxx'
}
}
Args:
exception (Type): Exception class expected to be raised.
msg (msg): Expected error message.
condition (dict): The parameter of search condition.
"""
self.assertRaisesRegex(
MindInsightException,
"The parameter metric_attribute is invalid. "
"It should be a dict and the value should be a float or a integer",
exception,
msg,
validate_search_model_condition,
SearchModelConditionParameter,
condition
)

def _assert_raise_of_mindinsight_exception(self, msg, condition):
"""
Assert raise of MindinsightException by unittest.

Args:
msg (msg): Expected error message.
condition (dict): The parameter of search condition.
"""
self._assert_raise(MindInsightException, msg, condition)

def _assert_raise_of_lineage_param_value_error(self, msg, condition):
"""
Assert raise of LineageParamValueError by unittest.

Args:
msg (msg): Expected error message.
condition (dict): The parameter of search condition.
"""
self._assert_raise(LineageParamValueError, msg, condition)

def _assert_raise_of_lineage_param_type_error(self, msg, condition):
"""
Assert raise of LineageParamTypeError by unittest.

Args:
msg (msg): Expected error message.
condition (dict): The parameter of search condition.
"""
self._assert_raise(LineageParamTypeError, msg, condition)

+ 4
- 46
tests/ut/lineagemgr/querier/event_data.py View File

@@ -15,6 +15,9 @@
"""The event data in querier test."""
import json

from ....utils.mindspore.dataset.engine.serializer_deserializer import \
SERIALIZED_PIPELINE

EVENT_TRAIN_DICT_0 = {
'wall_time': 1581499557.7017336,
'train_lineage': {
@@ -373,49 +376,4 @@ EVENT_DATASET_DICT_0 = {
}
}

DATASET_DICT_0 = {
'op_type': 'BatchDataset',
'op_module': 'minddata.dataengine.datasets',
'num_parallel_workers': None,
'drop_remainder': True,
'batch_size': 10,
'children': [
{
'op_type': 'MapDataset',
'op_module': 'minddata.dataengine.datasets',
'num_parallel_workers': None,
'input_columns': [
'label'
],
'output_columns': [
None
],
'operations': [
{
'tensor_op_module': 'minddata.transforms.c_transforms',
'tensor_op_name': 'OneHot',
'num_classes': 10
}
],
'children': [
{
'op_type': 'MnistDataset',
'shard_id': None,
'num_shards': None,
'op_module': 'minddata.dataengine.datasets',
'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData',
'num_parallel_workers': None,
'shuffle': None,
'num_samples': 100,
'sampler': {
'sampler_module': 'minddata.dataengine.samplers',
'sampler_name': 'RandomSampler',
'replacement': True,
'num_samples': 100
},
'children': []
}
]
}
]
}
DATASET_DICT_0 = SERIALIZED_PIPELINE

+ 129
- 356
tests/ut/lineagemgr/querier/test_querier.py View File

@@ -140,6 +140,98 @@ def get_lineage_infos():
return lineage_infos


LINEAGE_INFO_0 = {
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
LINEAGE_INFO_1 = {
'summary_dir': '/path/to/summary1',
**event_data.EVENT_TRAIN_DICT_1['train_lineage'],
'metric': event_data.METRIC_1,
'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
LINEAGE_FILTRATION_0 = create_filtration_result(
'/path/to/summary0',
event_data.EVENT_TRAIN_DICT_0,
event_data.EVENT_EVAL_DICT_0,
event_data.METRIC_0,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_1 = create_filtration_result(
'/path/to/summary1',
event_data.EVENT_TRAIN_DICT_1,
event_data.EVENT_EVAL_DICT_1,
event_data.METRIC_1,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_2 = create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_3 = create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_4 = create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_5 = {
"summary_dir": '/path/to/summary5',
"loss_function":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'],
"train_dataset_path": None,
"train_dataset_count":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'],
"test_dataset_path": None,
"test_dataset_count": None,
"network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'],
"optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'],
"learning_rate":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'],
"epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'],
"batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'],
"loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'],
"model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'],
"metric": {},
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
}
LINEAGE_FILTRATION_6 = {
"summary_dir": '/path/to/summary6',
"loss_function": None,
"train_dataset_path": None,
"train_dataset_count": None,
"test_dataset_path":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'],
"test_dataset_count":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'],
"network": None,
"optimizer": None,
"learning_rate": None,
"epoch": None,
"batch_size": None,
"loss": None,
"model_size": None,
"metric": event_data.METRIC_5,
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
}


class TestQuerier(TestCase):
"""Test the class of `Querier`."""
@mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos')
@@ -169,31 +261,13 @@ class TestQuerier(TestCase):

def test_get_summary_lineage_success_1(self):
"""Test the success of get_summary_lineage."""
expected_result = [
{
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
]
expected_result = [LINEAGE_INFO_0]
result = self.single_querier.get_summary_lineage()
self.assertListEqual(expected_result, result)

def test_get_summary_lineage_success_2(self):
"""Test the success of get_summary_lineage."""
expected_result = [
{
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset':
event_data.EVENT_EVAL_DICT_0['evaluation_lineage'][
'valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
]
expected_result = [LINEAGE_INFO_0]
result = self.single_querier.get_summary_lineage(
summary_dir='/path/to/summary0'
)
@@ -216,20 +290,8 @@ class TestQuerier(TestCase):
def test_get_summary_lineage_success_4(self):
"""Test the success of get_summary_lineage."""
expected_result = [
{
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
},
{
'summary_dir': '/path/to/summary1',
**event_data.EVENT_TRAIN_DICT_1['train_lineage'],
'metric': event_data.METRIC_1,
'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
},
LINEAGE_INFO_0,
LINEAGE_INFO_1,
{
'summary_dir': '/path/to/summary2',
**event_data.EVENT_TRAIN_DICT_2['train_lineage'],
@@ -274,15 +336,7 @@ class TestQuerier(TestCase):

def test_get_summary_lineage_success_5(self):
"""Test the success of get_summary_lineage."""
expected_result = [
{
'summary_dir': '/path/to/summary1',
**event_data.EVENT_TRAIN_DICT_1['train_lineage'],
'metric': event_data.METRIC_1,
'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
]
expected_result = [LINEAGE_INFO_1]
result = self.multi_querier.get_summary_lineage(
summary_dir='/path/to/summary1'
)
@@ -341,20 +395,8 @@ class TestQuerier(TestCase):
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary1',
event_data.EVENT_TRAIN_DICT_1,
event_data.EVENT_EVAL_DICT_1,
event_data.METRIC_1,
event_data.DATASET_DICT_0,
),
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_1,
LINEAGE_FILTRATION_2
],
'count': 2,
}
@@ -377,20 +419,8 @@ class TestQuerier(TestCase):
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_2,
LINEAGE_FILTRATION_3
],
'count': 2,
}
@@ -405,20 +435,8 @@ class TestQuerier(TestCase):
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
)
LINEAGE_FILTRATION_2,
LINEAGE_FILTRATION_3
],
'count': 7,
}
@@ -429,82 +447,13 @@ class TestQuerier(TestCase):
"""Test the success of filter_summary_lineage."""
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary0',
event_data.EVENT_TRAIN_DICT_0,
event_data.EVENT_EVAL_DICT_0,
event_data.METRIC_0,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary1',
event_data.EVENT_TRAIN_DICT_1,
event_data.EVENT_EVAL_DICT_1,
event_data.METRIC_1,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
),
{
"summary_dir": '/path/to/summary5',
"loss_function":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'],
"train_dataset_path": None,
"train_dataset_count":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'],
"test_dataset_path": None,
"test_dataset_count": None,
"network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'],
"optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'],
"learning_rate":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'],
"epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'],
"batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'],
"loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'],
"model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'],
"metric": {},
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
},
{
"summary_dir": '/path/to/summary6',
"loss_function": None,
"train_dataset_path": None,
"train_dataset_count": None,
"test_dataset_path":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'],
"test_dataset_count":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'],
"network": None,
"optimizer": None,
"learning_rate": None,
"epoch": None,
"batch_size": None,
"loss": None,
"model_size": None,
"metric": event_data.METRIC_5,
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
}
LINEAGE_FILTRATION_0,
LINEAGE_FILTRATION_1,
LINEAGE_FILTRATION_2,
LINEAGE_FILTRATION_3,
LINEAGE_FILTRATION_4,
LINEAGE_FILTRATION_5,
LINEAGE_FILTRATION_6
],
'count': 7,
}
@@ -519,15 +468,7 @@ class TestQuerier(TestCase):
}
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
),
],
'object': [LINEAGE_FILTRATION_4],
'count': 1,
}
result = self.multi_querier.filter_summary_lineage(condition=condition)
@@ -541,82 +482,13 @@ class TestQuerier(TestCase):
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary0',
event_data.EVENT_TRAIN_DICT_0,
event_data.EVENT_EVAL_DICT_0,
event_data.METRIC_0,
event_data.DATASET_DICT_0
),
{
"summary_dir": '/path/to/summary5',
"loss_function":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'],
"train_dataset_path": None,
"train_dataset_count":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'],
"test_dataset_path": None,
"test_dataset_count": None,
"network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'],
"optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'],
"learning_rate":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'],
"epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'],
"batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'],
"loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'],
"model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'],
"metric": {},
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
},
create_filtration_result(
'/path/to/summary1',
event_data.EVENT_TRAIN_DICT_1,
event_data.EVENT_EVAL_DICT_1,
event_data.METRIC_1,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
),
{
"summary_dir": '/path/to/summary6',
"loss_function": None,
"train_dataset_path": None,
"train_dataset_count": None,
"test_dataset_path":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'],
"test_dataset_count":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'],
"network": None,
"optimizer": None,
"learning_rate": None,
"epoch": None,
"batch_size": None,
"loss": None,
"model_size": None,
"metric": event_data.METRIC_5,
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
}
LINEAGE_FILTRATION_0,
LINEAGE_FILTRATION_5,
LINEAGE_FILTRATION_1,
LINEAGE_FILTRATION_2,
LINEAGE_FILTRATION_3,
LINEAGE_FILTRATION_4,
LINEAGE_FILTRATION_6
],
'count': 7,
}
@@ -631,82 +503,13 @@ class TestQuerier(TestCase):
}
expected_result = {
'object': [
{
"summary_dir": '/path/to/summary6',
"loss_function": None,
"train_dataset_path": None,
"train_dataset_count": None,
"test_dataset_path":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'],
"test_dataset_count":
event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'],
"network": None,
"optimizer": None,
"learning_rate": None,
"epoch": None,
"batch_size": None,
"loss": None,
"model_size": None,
"metric": event_data.METRIC_5,
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
},
create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary3',
event_data.EVENT_TRAIN_DICT_3,
event_data.EVENT_EVAL_DICT_3,
event_data.METRIC_3,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary2',
event_data.EVENT_TRAIN_DICT_2,
event_data.EVENT_EVAL_DICT_2,
event_data.METRIC_2,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary1',
event_data.EVENT_TRAIN_DICT_1,
event_data.EVENT_EVAL_DICT_1,
event_data.METRIC_1,
event_data.DATASET_DICT_0
),
create_filtration_result(
'/path/to/summary0',
event_data.EVENT_TRAIN_DICT_0,
event_data.EVENT_EVAL_DICT_0,
event_data.METRIC_0,
event_data.DATASET_DICT_0
),
{
"summary_dir": '/path/to/summary5',
"loss_function":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'],
"train_dataset_path": None,
"train_dataset_count":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'],
"test_dataset_path": None,
"test_dataset_count": None,
"network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'],
"optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'],
"learning_rate":
event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'],
"epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'],
"batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'],
"loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'],
"model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'],
"metric": {},
"dataset_graph": event_data.DATASET_DICT_0,
"dataset_mark": '2'
}
LINEAGE_FILTRATION_6,
LINEAGE_FILTRATION_4,
LINEAGE_FILTRATION_3,
LINEAGE_FILTRATION_2,
LINEAGE_FILTRATION_1,
LINEAGE_FILTRATION_0,
LINEAGE_FILTRATION_5
],
'count': 7,
}
@@ -722,15 +525,7 @@ class TestQuerier(TestCase):
}
}
expected_result = {
'object': [
create_filtration_result(
'/path/to/summary4',
event_data.EVENT_TRAIN_DICT_4,
event_data.EVENT_EVAL_DICT_4,
event_data.METRIC_4,
event_data.DATASET_DICT_0
),
],
'object': [LINEAGE_FILTRATION_4],
'count': 1,
}
result = self.multi_querier.filter_summary_lineage(condition=condition)
@@ -809,20 +604,8 @@ class TestQuerier(TestCase):
querier = Querier(summary_path)
querier._parse_failed_paths.append('/path/to/summary1/log1')
expected_result = [
{
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
},
{
'summary_dir': '/path/to/summary1',
**event_data.EVENT_TRAIN_DICT_1['train_lineage'],
'metric': event_data.METRIC_1,
'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
LINEAGE_INFO_0,
LINEAGE_INFO_1
]
result = querier.get_summary_lineage()
self.assertListEqual(expected_result, result)
@@ -842,17 +625,7 @@ class TestQuerier(TestCase):
querier._parse_failed_paths.append('/path/to/summary1/log1')

args[0].return_value = create_lineage_info(None, None, None)
expected_result = [
{
'summary_dir': '/path/to/summary0',
**event_data.EVENT_TRAIN_DICT_0['train_lineage'],
'metric': event_data.METRIC_0,
'valid_dataset':
event_data.EVENT_EVAL_DICT_0['evaluation_lineage'][
'valid_dataset'],
'dataset_graph': event_data.DATASET_DICT_0
}
]
expected_result = [LINEAGE_INFO_0]
result = querier.get_summary_lineage()
self.assertListEqual(expected_result, result)
self.assertListEqual(


Write Preview
Loading…
Cancel
Save