Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: e497117b74
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e497117b74'
${ noResults }
mindspore/tests/ut/python/dataset/test_minddataset_sampler.py

629 lines
27 kB
Raw Blame History 

  1. # Copyright 2019 Huawei Technologies Co., Ltd

  2. #

  3. # Licensed under the Apache License, Version 2.0 (the "License");

  4. # you may not use this file except in compliance with the License.

  5. # You may obtain a copy of the License at

  6. #

  7. # http://www.apache.org/licenses/LICENSE-2.0

  8. #

  9. # Unless required by applicable law or agreed to in writing, software

  10. # distributed under the License is distributed on an "AS IS" BASIS,

  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  12. # See the License for the specific language governing permissions and

  13. # limitations under the License.

  14. # ==============================================================================

  15. """

  16. This is the test module for mindrecord

  17. """

  18. import os

  19. import pytest

  20. import numpy as np

  21. 

  22. import mindspore.dataset as ds

  23. from mindspore import log as logger

  24. from mindspore.dataset.text import to_str

  25. from mindspore.mindrecord import FileWriter

  26. 

  27. FILES_NUM = 4

  28. CV_FILE_NAME = "../data/mindrecord/imagenet.mindrecord"

  29. CV_DIR_NAME = "../data/mindrecord/testImageNetData"

  30. 

  31. 

  32. @pytest.fixture

  33. def add_and_remove_cv_file():

  34.     """add/remove cv file"""

  35.     paths = ["{}{}".format(CV_FILE_NAME, str(x).rjust(1, '0'))

  36.              for x in range(FILES_NUM)]

  37.     for x in paths:

  38.         if os.path.exists("{}".format(x)):

  39.             os.remove("{}".format(x))

  40.         if os.path.exists("{}.db".format(x)):

  41.             os.remove("{}.db".format(x))

  42.     writer = FileWriter(CV_FILE_NAME, FILES_NUM)

  43.     data = get_data(CV_DIR_NAME, True)

  44.     cv_schema_json = {"id": {"type": "int32"},

  45.                       "file_name": {"type": "string"},

  46.                       "label": {"type": "int32"},

  47.                       "data": {"type": "bytes"}}

  48.     writer.add_schema(cv_schema_json, "img_schema")

  49.     writer.add_index(["file_name", "label"])

  50.     writer.write_raw_data(data)

  51.     writer.commit()

  52.     yield "yield_cv_data"

  53.     for x in paths:

  54.         os.remove("{}".format(x))

  55.         os.remove("{}.db".format(x))

  56. 

  57. 

  58. def test_cv_minddataset_pk_sample_no_column(add_and_remove_cv_file):

  59.     """tutorial for cv minderdataset."""

  60.     num_readers = 4

  61.     sampler = ds.PKSampler(2)

  62.     data_set = ds.MindDataset(CV_FILE_NAME + "0", None, num_readers,

  63.                               sampler=sampler)

  64. 

  65.     assert data_set.get_dataset_size() == 6

  66.     num_iter = 0

  67.     for item in data_set.create_dict_iterator():

  68.         logger.info(

  69.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  70.         logger.info("-------------- item[file_name]: \

  71.                 {}------------------------".format(to_str(item["file_name"])))

  72.         logger.info(

  73.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  74.         num_iter += 1

  75. 

  76. 

  77. def test_cv_minddataset_pk_sample_basic(add_and_remove_cv_file):

  78.     """tutorial for cv minderdataset."""

  79.     columns_list = ["data", "file_name", "label"]

  80.     num_readers = 4

  81.     sampler = ds.PKSampler(2)

  82.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  83.                               sampler=sampler)

  84. 

  85.     assert data_set.get_dataset_size() == 6

  86.     num_iter = 0

  87.     for item in data_set.create_dict_iterator():

  88.         logger.info(

  89.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  90.         logger.info("-------------- item[data]: \

  91.                 {}------------------------".format(item["data"][:10]))

  92.         logger.info("-------------- item[file_name]: \

  93.                 {}------------------------".format(to_str(item["file_name"])))

  94.         logger.info(

  95.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  96.         num_iter += 1

  97. 

  98. 

  99. def test_cv_minddataset_pk_sample_shuffle(add_and_remove_cv_file):

  100.     """tutorial for cv minderdataset."""

  101.     columns_list = ["data", "file_name", "label"]

  102.     num_readers = 4

  103.     sampler = ds.PKSampler(3, None, True)

  104.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  105.                               sampler=sampler)

  106. 

  107.     assert data_set.get_dataset_size() == 9

  108.     num_iter = 0

  109.     for item in data_set.create_dict_iterator():

  110.         logger.info(

  111.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  112.         logger.info("-------------- item[file_name]: \

  113.                 {}------------------------".format(to_str(item["file_name"])))

  114.         logger.info(

  115.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  116.         num_iter += 1

  117. 

  118. 

  119. def test_cv_minddataset_pk_sample_out_of_range(add_and_remove_cv_file):

  120.     """tutorial for cv minderdataset."""

  121.     columns_list = ["data", "file_name", "label"]

  122.     num_readers = 4

  123.     sampler = ds.PKSampler(5, None, True)

  124.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  125.                               sampler=sampler)

  126.     assert data_set.get_dataset_size() == 15

  127.     num_iter = 0

  128.     for item in data_set.create_dict_iterator():

  129.         logger.info(

  130.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  131.         logger.info("-------------- item[file_name]: \

  132.                 {}------------------------".format(to_str(item["file_name"])))

  133.         logger.info(

  134.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  135.         num_iter += 1

  136. 

  137. 

  138. def test_cv_minddataset_subset_random_sample_basic(add_and_remove_cv_file):

  139.     """tutorial for cv minderdataset."""

  140.     columns_list = ["data", "file_name", "label"]

  141.     num_readers = 4

  142.     indices = [1, 2, 3, 5, 7]

  143.     sampler = ds.SubsetRandomSampler(indices)

  144.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  145.                               sampler=sampler)

  146.     assert data_set.get_dataset_size() == 5

  147.     num_iter = 0

  148.     for item in data_set.create_dict_iterator():

  149.         logger.info(

  150.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  151.         logger.info(

  152.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  153.         logger.info(

  154.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  155.         logger.info(

  156.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  157.         num_iter += 1

  158.     assert num_iter == 5

  159. 

  160. 

  161. def test_cv_minddataset_subset_random_sample_replica(add_and_remove_cv_file):

  162.     """tutorial for cv minderdataset."""

  163.     columns_list = ["data", "file_name", "label"]

  164.     num_readers = 4

  165.     indices = [1, 2, 2, 5, 7, 9]

  166.     sampler = ds.SubsetRandomSampler(indices)

  167.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  168.                               sampler=sampler)

  169.     assert data_set.get_dataset_size() == 6

  170.     num_iter = 0

  171.     for item in data_set.create_dict_iterator():

  172.         logger.info(

  173.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  174.         logger.info(

  175.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  176.         logger.info(

  177.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  178.         logger.info(

  179.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  180.         num_iter += 1

  181.     assert num_iter == 6

  182. 

  183. 

  184. def test_cv_minddataset_subset_random_sample_empty(add_and_remove_cv_file):

  185.     """tutorial for cv minderdataset."""

  186.     columns_list = ["data", "file_name", "label"]

  187.     num_readers = 4

  188.     indices = []

  189.     sampler = ds.SubsetRandomSampler(indices)

  190.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  191.                               sampler=sampler)

  192.     assert data_set.get_dataset_size() == 0

  193.     num_iter = 0

  194.     for item in data_set.create_dict_iterator():

  195.         logger.info(

  196.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  197.         logger.info(

  198.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  199.         logger.info(

  200.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  201.         logger.info(

  202.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  203.         num_iter += 1

  204.     assert num_iter == 0

  205. 

  206. 

  207. def test_cv_minddataset_subset_random_sample_out_of_range(add_and_remove_cv_file):

  208.     """tutorial for cv minderdataset."""

  209.     columns_list = ["data", "file_name", "label"]

  210.     num_readers = 4

  211.     indices = [1, 2, 4, 11, 13]

  212.     sampler = ds.SubsetRandomSampler(indices)

  213.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  214.                               sampler=sampler)

  215.     assert data_set.get_dataset_size() == 5

  216.     num_iter = 0

  217.     for item in data_set.create_dict_iterator():

  218.         logger.info(

  219.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  220.         logger.info(

  221.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  222.         logger.info(

  223.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  224.         logger.info(

  225.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  226.         num_iter += 1

  227.     assert num_iter == 5

  228. 

  229. 

  230. def test_cv_minddataset_subset_random_sample_negative(add_and_remove_cv_file):

  231.     columns_list = ["data", "file_name", "label"]

  232.     num_readers = 4

  233.     indices = [1, 2, 4, -1, -2]

  234.     sampler = ds.SubsetRandomSampler(indices)

  235.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  236.                               sampler=sampler)

  237.     assert data_set.get_dataset_size() == 5

  238.     num_iter = 0

  239.     for item in data_set.create_dict_iterator():

  240.         logger.info(

  241.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  242.         logger.info(

  243.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  244.         logger.info(

  245.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  246.         logger.info(

  247.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  248.         num_iter += 1

  249.     assert num_iter == 5

  250. 

  251. 

  252. def test_cv_minddataset_random_sampler_basic(add_and_remove_cv_file):

  253.     data = get_data(CV_DIR_NAME, True)

  254.     columns_list = ["data", "file_name", "label"]

  255.     num_readers = 4

  256.     sampler = ds.RandomSampler()

  257.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  258.                               sampler=sampler)

  259.     assert data_set.get_dataset_size() == 10

  260.     num_iter = 0

  261.     new_dataset = []

  262.     for item in data_set.create_dict_iterator():

  263.         logger.info(

  264.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  265.         logger.info(

  266.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  267.         logger.info(

  268.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  269.         logger.info(

  270.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  271.         num_iter += 1

  272.         new_dataset.append(item['file_name'])

  273.     assert num_iter == 10

  274.     assert new_dataset != [x['file_name'] for x in data]

  275. 

  276. def test_cv_minddataset_random_sampler_repeat(add_and_remove_cv_file):

  277.     columns_list = ["data", "file_name", "label"]

  278.     num_readers = 4

  279.     sampler = ds.RandomSampler()

  280.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  281.                               sampler=sampler)

  282.     assert data_set.get_dataset_size() == 10

  283.     ds1 = data_set.repeat(3)

  284.     num_iter = 0

  285.     epoch1_dataset = []

  286.     epoch2_dataset = []

  287.     epoch3_dataset = []

  288.     for item in ds1.create_dict_iterator():

  289.         logger.info(

  290.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  291.         logger.info(

  292.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  293.         logger.info(

  294.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  295.         logger.info(

  296.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  297.         num_iter += 1

  298.         if num_iter <= 10:

  299.             epoch1_dataset.append(item['file_name'])

  300.         elif num_iter <= 20:

  301.             epoch2_dataset.append(item['file_name'])

  302.         else:

  303.             epoch3_dataset.append(item['file_name'])

  304.     assert num_iter == 30

  305.     assert epoch1_dataset not in (epoch2_dataset, epoch3_dataset)

  306.     assert epoch2_dataset not in (epoch1_dataset, epoch3_dataset)

  307.     assert epoch3_dataset not in (epoch1_dataset, epoch2_dataset)

  308. 

  309. def test_cv_minddataset_random_sampler_replacement(add_and_remove_cv_file):

  310.     columns_list = ["data", "file_name", "label"]

  311.     num_readers = 4

  312.     sampler = ds.RandomSampler(replacement=True, num_samples=5)

  313.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  314.                               sampler=sampler)

  315.     assert data_set.get_dataset_size() == 5

  316.     num_iter = 0

  317.     for item in data_set.create_dict_iterator():

  318.         logger.info(

  319.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  320.         logger.info(

  321.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  322.         logger.info(

  323.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  324.         logger.info(

  325.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  326.         num_iter += 1

  327.     assert num_iter == 5

  328. 

  329. 

  330. def test_cv_minddataset_sequential_sampler_basic(add_and_remove_cv_file):

  331.     data = get_data(CV_DIR_NAME, True)

  332.     columns_list = ["data", "file_name", "label"]

  333.     num_readers = 4

  334.     sampler = ds.SequentialSampler(1, 4)

  335.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  336.                               sampler=sampler)

  337.     assert data_set.get_dataset_size() == 4

  338.     num_iter = 0

  339.     for item in data_set.create_dict_iterator():

  340.         logger.info(

  341.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  342.         logger.info(

  343.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  344.         logger.info(

  345.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  346.         logger.info(

  347.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  348.         assert item['file_name'] == np.array(

  349.             data[num_iter+1]['file_name'], dtype='S')

  350.         num_iter += 1

  351.     assert num_iter == 4

  352. 

  353. 

  354. def test_cv_minddataset_sequential_sampler_exceed_size(add_and_remove_cv_file):

  355.     data = get_data(CV_DIR_NAME, True)

  356.     columns_list = ["data", "file_name", "label"]

  357.     num_readers = 4

  358.     sampler = ds.SequentialSampler(2, 10)

  359.     data_set = ds.MindDataset(CV_FILE_NAME + "0", columns_list, num_readers,

  360.                               sampler=sampler)

  361.     dataset_size = data_set.get_dataset_size()

  362.     assert dataset_size == 10

  363.     num_iter = 0

  364.     for item in data_set.create_dict_iterator():

  365.         logger.info(

  366.             "-------------- cv reader basic: {} ------------------------".format(num_iter))

  367.         logger.info(

  368.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  369.         logger.info(

  370.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  371.         logger.info(

  372.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  373.         assert item['file_name'] == np.array(

  374.             data[(num_iter + 2) % dataset_size]['file_name'], dtype='S')

  375.         num_iter += 1

  376.     assert num_iter == 10

  377. 

  378. 

  379. def test_cv_minddataset_split_basic(add_and_remove_cv_file):

  380.     data = get_data(CV_DIR_NAME, True)

  381.     columns_list = ["data", "file_name", "label"]

  382.     num_readers = 4

  383.     d = ds.MindDataset(CV_FILE_NAME + "0", columns_list,

  384.                        num_readers, shuffle=False)

  385.     d1, d2 = d.split([8, 2], randomize=False)

  386.     assert d.get_dataset_size() == 10

  387.     assert d1.get_dataset_size() == 8

  388.     assert d2.get_dataset_size() == 2

  389.     num_iter = 0

  390.     for item in d1.create_dict_iterator():

  391.         logger.info(

  392.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  393.         logger.info(

  394.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  395.         logger.info(

  396.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  397.         assert item['file_name'] == np.array(data[num_iter]['file_name'],

  398.                                              dtype='S')

  399.         num_iter += 1

  400.     assert num_iter == 8

  401.     num_iter = 0

  402.     for item in d2.create_dict_iterator():

  403.         logger.info(

  404.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  405.         logger.info(

  406.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  407.         logger.info(

  408.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  409.         assert item['file_name'] == np.array(data[num_iter + 8]['file_name'],

  410.                                              dtype='S')

  411.         num_iter += 1

  412.     assert num_iter == 2

  413. 

  414. 

  415. def test_cv_minddataset_split_exact_percent(add_and_remove_cv_file):

  416.     data = get_data(CV_DIR_NAME, True)

  417.     columns_list = ["data", "file_name", "label"]

  418.     num_readers = 4

  419.     d = ds.MindDataset(CV_FILE_NAME + "0", columns_list,

  420.                        num_readers, shuffle=False)

  421.     d1, d2 = d.split([0.8, 0.2], randomize=False)

  422.     assert d.get_dataset_size() == 10

  423.     assert d1.get_dataset_size() == 8

  424.     assert d2.get_dataset_size() == 2

  425.     num_iter = 0

  426.     for item in d1.create_dict_iterator():

  427.         logger.info(

  428.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  429.         logger.info(

  430.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  431.         logger.info(

  432.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  433.         assert item['file_name'] == np.array(

  434.             data[num_iter]['file_name'], dtype='S')

  435.         num_iter += 1

  436.     assert num_iter == 8

  437.     num_iter = 0

  438.     for item in d2.create_dict_iterator():

  439.         logger.info(

  440.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  441.         logger.info(

  442.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  443.         logger.info(

  444.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  445.         assert item['file_name'] == np.array(data[num_iter + 8]['file_name'],

  446.                                              dtype='S')

  447.         num_iter += 1

  448.     assert num_iter == 2

  449. 

  450. 

  451. def test_cv_minddataset_split_fuzzy_percent(add_and_remove_cv_file):

  452.     data = get_data(CV_DIR_NAME, True)

  453.     columns_list = ["data", "file_name", "label"]

  454.     num_readers = 4

  455.     d = ds.MindDataset(CV_FILE_NAME + "0", columns_list,

  456.                        num_readers, shuffle=False)

  457.     d1, d2 = d.split([0.41, 0.59], randomize=False)

  458.     assert d.get_dataset_size() == 10

  459.     assert d1.get_dataset_size() == 4

  460.     assert d2.get_dataset_size() == 6

  461.     num_iter = 0

  462.     for item in d1.create_dict_iterator():

  463.         logger.info(

  464.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  465.         logger.info(

  466.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  467.         logger.info(

  468.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  469.         assert item['file_name'] == np.array(

  470.             data[num_iter]['file_name'], dtype='S')

  471.         num_iter += 1

  472.     assert num_iter == 4

  473.     num_iter = 0

  474.     for item in d2.create_dict_iterator():

  475.         logger.info(

  476.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  477.         logger.info(

  478.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  479.         logger.info(

  480.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  481.         assert item['file_name'] == np.array(data[num_iter + 4]['file_name'],

  482.                                              dtype='S')

  483.         num_iter += 1

  484.     assert num_iter == 6

  485. 

  486. 

  487. def test_cv_minddataset_split_deterministic(add_and_remove_cv_file):

  488.     columns_list = ["data", "file_name", "label"]

  489.     num_readers = 4

  490.     d = ds.MindDataset(CV_FILE_NAME + "0", columns_list,

  491.                        num_readers, shuffle=False)

  492.     # should set seed to avoid data overlap

  493.     ds.config.set_seed(111)

  494.     d1, d2 = d.split([0.8, 0.2])

  495.     assert d.get_dataset_size() == 10

  496.     assert d1.get_dataset_size() == 8

  497.     assert d2.get_dataset_size() == 2

  498. 

  499.     d1_dataset = []

  500.     d2_dataset = []

  501.     num_iter = 0

  502.     for item in d1.create_dict_iterator():

  503.         logger.info(

  504.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  505.         logger.info(

  506.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  507.         logger.info(

  508.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  509.         d1_dataset.append(item['file_name'])

  510.         num_iter += 1

  511.     assert num_iter == 8

  512.     num_iter = 0

  513.     for item in d2.create_dict_iterator():

  514.         logger.info(

  515.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  516.         logger.info(

  517.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  518.         logger.info(

  519.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  520.         d2_dataset.append(item['file_name'])

  521.         num_iter += 1

  522.     assert num_iter == 2

  523.     inter_dataset = [x for x in d1_dataset if x in d2_dataset]

  524.     assert inter_dataset == []   # intersection of  d1 and d2

  525. 

  526. 

  527. def test_cv_minddataset_split_sharding(add_and_remove_cv_file):

  528.     data = get_data(CV_DIR_NAME, True)

  529.     columns_list = ["data", "file_name", "label"]

  530.     num_readers = 4

  531.     d = ds.MindDataset(CV_FILE_NAME + "0", columns_list,

  532.                        num_readers, shuffle=False)

  533.     # should set seed to avoid data overlap

  534.     ds.config.set_seed(111)

  535.     d1, d2 = d.split([0.8, 0.2])

  536.     assert d.get_dataset_size() == 10

  537.     assert d1.get_dataset_size() == 8

  538.     assert d2.get_dataset_size() == 2

  539.     distributed_sampler = ds.DistributedSampler(2, 0)

  540.     d1.use_sampler(distributed_sampler)

  541.     assert d1.get_dataset_size() == 4

  542. 

  543.     num_iter = 0

  544.     d1_shard1 = []

  545.     for item in d1.create_dict_iterator():

  546.         logger.info(

  547.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  548.         logger.info(

  549.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  550.         logger.info(

  551.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  552.         num_iter += 1

  553.         d1_shard1.append(item['file_name'])

  554.     assert num_iter == 4

  555.     assert d1_shard1 != [x['file_name'] for x in data[0:4]]

  556. 

  557.     distributed_sampler = ds.DistributedSampler(2, 1)

  558.     d1.use_sampler(distributed_sampler)

  559.     assert d1.get_dataset_size() == 4

  560. 

  561.     d1s = d1.repeat(3)

  562.     epoch1_dataset = []

  563.     epoch2_dataset = []

  564.     epoch3_dataset = []

  565.     num_iter = 0

  566.     for item in d1s.create_dict_iterator():

  567.         logger.info(

  568.             "-------------- item[data]: {}  -----------------------------".format(item["data"]))

  569.         logger.info(

  570.             "-------------- item[file_name]: {} ------------------------".format(item["file_name"]))

  571.         logger.info(

  572.             "-------------- item[label]: {} ----------------------------".format(item["label"]))

  573.         num_iter += 1

  574.         if num_iter <= 4:

  575.             epoch1_dataset.append(item['file_name'])

  576.         elif num_iter <= 8:

  577.             epoch2_dataset.append(item['file_name'])

  578.         else:

  579.             epoch3_dataset.append(item['file_name'])

  580.     assert len(epoch1_dataset) == 4

  581.     assert len(epoch2_dataset) == 4

  582.     assert len(epoch3_dataset) == 4

  583.     inter_dataset = [x for x in d1_shard1 if x in epoch1_dataset]

  584.     assert inter_dataset == [] # intersection of d1's shard1 and d1's shard2

  585.     assert epoch1_dataset not in (epoch2_dataset, epoch3_dataset)

  586.     assert epoch2_dataset not in (epoch1_dataset, epoch3_dataset)

  587.     assert epoch3_dataset not in (epoch1_dataset, epoch2_dataset)

  588. 

  589.     epoch1_dataset.sort()

  590.     epoch2_dataset.sort()

  591.     epoch3_dataset.sort()

  592.     assert epoch1_dataset != epoch2_dataset

  593.     assert epoch2_dataset != epoch3_dataset

  594.     assert epoch3_dataset != epoch1_dataset

  595. 

  596. 

  597. def get_data(dir_name, sampler=False):

  598.     """

  599.     usage: get data from imagenet dataset

  600.     params:

  601.     dir_name: directory containing folder images and annotation information

  602. 

  603.     """

  604.     if not os.path.isdir(dir_name):

  605.         raise IOError("Directory {} not exists".format(dir_name))

  606.     img_dir = os.path.join(dir_name, "images")

  607.     if sampler:

  608.         ann_file = os.path.join(dir_name, "annotation_sampler.txt")

  609.     else:

  610.         ann_file = os.path.join(dir_name, "annotation.txt")

  611.     with open(ann_file, "r") as file_reader:

  612.         lines = file_reader.readlines()

  613. 

  614.     data_list = []

  615.     for i, line in enumerate(lines):

  616.         try:

  617.             filename, label = line.split(",")

  618.             label = label.strip("\n")

  619.             with open(os.path.join(img_dir, filename), "rb") as file_reader:

  620.                 img = file_reader.read()

  621.             data_json = {"id": i,

  622.                          "file_name": filename,

  623.                          "data": img,

  624.                          "label": int(label)}

  625.             data_list.append(data_json)

  626.         except FileNotFoundError:

  627.             continue

  628.     return data_list