Adaptation for new api of MS.Dataset

mindarmour/fuzz_testing/image_transform.py

@@ -17,7 +17,7 @@ Image transform
 import numpy as np
 from PIL import Image, ImageEnhance, ImageFilter
 
-from mindspore.dataset.transforms.vision.py_transforms_util import is_numpy, \
+from mindspore.dataset.vision.py_transforms_util import is_numpy, \
     to_pil, hwc_to_chw
 from mindarmour.utils._check_param import check_param_multi_types
 from mindarmour.utils.logger import LogUtil

mindarmour/privacy/diff_privacy/mechanisms/mechanisms.py

@@ -144,7 +144,6 @@ class NoiseMechanismsFactory:
             >>>                 metrics=None)
             >>> ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
             >>>                            ['data', 'label'])
-            >>> ms_ds.set_dataset_size(batch_size * batches)
             >>> model.train(epochs, ms_ds, dataset_sink_mode=False)
         """
         if mech_name == 'Gaussian':

mindarmour/privacy/diff_privacy/train/model.py

@@ -114,7 +114,6 @@ class DPModel(Model):
         >>>                 metrics=None)
         >>> ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
         >>>                             ['data', 'label'])
-        >>> ms_ds.set_dataset_size(batch_size*batches)
         >>> model.train(epochs, ms_ds, dataset_sink_mode=False)
     """
 

tests/ut/python/privacy/diff_privacy/test_model_train.py

@@ -75,7 +75,6 @@ def test_dp_model_with_pynative_mode():
                     metrics=None)
     ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                                 ['data', 'label'])
-    ms_ds.set_dataset_size(batch_size*batches)
     model.train(epochs, ms_ds, dataset_sink_mode=False)
 
 
@@ -113,7 +112,6 @@ def test_dp_model_with_graph_mode():
                     metrics=None)
     ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                                 ['data', 'label'])
-    ms_ds.set_dataset_size(batch_size*batches)
     model.train(epochs, ms_ds, dataset_sink_mode=False)
 
 
@@ -150,5 +148,4 @@ def test_dp_model_with_graph_mode_ada_gaussian():
                     metrics=None)
     ms_ds = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                                 ['data', 'label'])
-    ms_ds.set_dataset_size(batch_size*batches)
     model.train(epochs, ms_ds, dataset_sink_mode=False)

tests/ut/python/privacy/diff_privacy/test_monitor.py

@@ -66,7 +66,6 @@ def test_dp_monitor():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[rdp], dataset_sink_mode=False)
 
 
@@ -95,7 +94,6 @@ def test_dp_monitor_gpu():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[rdp], dataset_sink_mode=False)
 
 
@@ -124,7 +122,6 @@ def test_dp_monitor_cpu():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[rdp], dataset_sink_mode=False)
 
 
@@ -154,7 +151,6 @@ def test_dp_monitor_zcdp():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[zcdp], dataset_sink_mode=False)
 
 
@@ -183,7 +179,6 @@ def test_dp_monitor_zcdp_gpu():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[zcdp], dataset_sink_mode=False)
 
 
@@ -212,5 +207,4 @@ def test_dp_monitor_zcdp_cpu():
     LOGGER.info(TAG, "============== Starting Training ==============")
     ds1 = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                               ["data", "label"])
-    ds1.set_dataset_size(batch_size * batches)
     model.train(epochs, ds1, callbacks=[zcdp], dataset_sink_mode=False)

tests/ut/python/privacy/evaluation/test_membership_inference.py

@@ -80,8 +80,6 @@ def test_membership_inference_object_train():
                                    ["image", "label"])
     ds_test = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                                   ["image", "label"])
-    ds_train.set_dataset_size(batch_size*batches)
-    ds_test.set_dataset_size((batch_size*batches))
     inference_model.train(ds_train, ds_test, config)
 
 
@@ -104,8 +102,6 @@ def test_membership_inference_eval():
                                      ["image", "label"])
     eval_test = ds.GeneratorDataset(dataset_generator(batch_size, batches),
                                     ["image", "label"])
-    eval_train.set_dataset_size(batch_size * batches)
-    eval_test.set_dataset_size((batch_size * batches))
 
     metrics = ["precision", "accuracy", "recall"]
     inference_model.eval(eval_train, eval_test, metrics)