Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: e497117b74
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e497117b74'
${ noResults }
mindspore/tests/ut/python/dataset/test_project.py

158 lines
5.9 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. from util import save_and_check_tuple

  16. 

  17. import mindspore.dataset as ds

  18. import mindspore.dataset.transforms.c_transforms as C

  19. from mindspore.common import dtype as mstype

  20. 

  21. DATA_DIR_TF = ["../data/dataset/testTFTestAllTypes/test.data"]

  22. SCHEMA_DIR_TF = "../data/dataset/testTFTestAllTypes/datasetSchema.json"

  23. GENERATE_GOLDEN = False

  24. 

  25. 

  26. def test_case_project_single_column():

  27.     columns = ["col_sint32"]

  28.     parameters = {"params": {'columns': columns}}

  29. 

  30.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  31.     data1 = data1.project(columns=columns)

  32. 

  33.     filename = "project_single_column_result.npz"

  34.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  35. 

  36. 

  37. def test_case_project_multiple_columns_in_order():

  38.     columns = ["col_sint16", "col_float", "col_2d"]

  39.     parameters = {"params": {'columns': columns}}

  40. 

  41.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  42.     data1 = data1.project(columns=columns)

  43. 

  44.     filename = "project_multiple_columns_in_order_result.npz"

  45.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  46. 

  47. 

  48. def test_case_project_multiple_columns_out_of_order():

  49.     columns = ["col_3d", "col_sint64", "col_2d"]

  50.     parameters = {"params": {'columns': columns}}

  51. 

  52.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  53.     data1 = data1.project(columns=columns)

  54. 

  55.     filename = "project_multiple_columns_out_of_order_result.npz"

  56.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  57. 

  58. 

  59. def test_case_project_map():

  60.     columns = ["col_3d", "col_sint64", "col_2d"]

  61.     parameters = {"params": {'columns': columns}}

  62. 

  63.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  64.     data1 = data1.project(columns=columns)

  65. 

  66.     type_cast_op = C.TypeCast(mstype.int64)

  67.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  68. 

  69.     filename = "project_map_after_result.npz"

  70.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  71. 

  72. 

  73. def test_case_map_project():

  74.     columns = ["col_3d", "col_sint64", "col_2d"]

  75.     parameters = {"params": {'columns': columns}}

  76. 

  77.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  78. 

  79.     type_cast_op = C.TypeCast(mstype.int64)

  80.     data1 = data1.map(input_columns=["col_sint64"], operations=type_cast_op)

  81. 

  82.     data1 = data1.project(columns=columns)

  83. 

  84.     filename = "project_map_before_result.npz"

  85.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  86. 

  87. 

  88. def test_case_project_between_maps():

  89.     columns = ["col_3d", "col_sint64", "col_2d"]

  90.     parameters = {"params": {'columns': columns}}

  91. 

  92.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  93. 

  94.     type_cast_op = C.TypeCast(mstype.int64)

  95.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  96.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  97.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  98.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  99. 

  100.     data1 = data1.project(columns=columns)

  101. 

  102.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  103.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  104.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  105.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  106.     data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)

  107. 

  108.     filename = "project_between_maps_result.npz"

  109.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  110. 

  111. 

  112. def test_case_project_repeat():

  113.     columns = ["col_3d", "col_sint64", "col_2d"]

  114.     parameters = {"params": {'columns': columns}}

  115. 

  116.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  117.     data1 = data1.project(columns=columns)

  118. 

  119.     repeat_count = 3

  120.     data1 = data1.repeat(repeat_count)

  121. 

  122.     filename = "project_before_repeat_result.npz"

  123.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  124. 

  125. 

  126. def test_case_repeat_project():

  127.     columns = ["col_3d", "col_sint64", "col_2d"]

  128.     parameters = {"params": {'columns': columns}}

  129. 

  130.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  131. 

  132.     repeat_count = 3

  133.     data1 = data1.repeat(repeat_count)

  134. 

  135.     data1 = data1.project(columns=columns)

  136. 

  137.     filename = "project_after_repeat_result.npz"

  138.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)

  139. 

  140. 

  141. def test_case_map_project_map_project():

  142.     columns = ["col_3d", "col_sint64", "col_2d"]

  143.     parameters = {"params": {'columns': columns}}

  144. 

  145.     data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)

  146. 

  147.     type_cast_op = C.TypeCast(mstype.int64)

  148.     data1 = data1.map(input_columns=["col_sint64"], operations=type_cast_op)

  149. 

  150.     data1 = data1.project(columns=columns)

  151. 

  152.     data1 = data1.map(input_columns=["col_2d"], operations=type_cast_op)

  153. 

  154.     data1 = data1.project(columns=columns)

  155. 

  156.     filename = "project_alternate_parallel_inline_result.npz"

  157.     save_and_check_tuple(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)