!86 Fix: Samples from previous steps are erased due to tags size too large

Merge pull request !86 from 李鸿章/fix_step_erased

mindinsight/datavisual/data_transform/events_data.py

@@ -17,10 +17,10 @@
 import collections
 import threading
 
+from mindinsight.conf import settings
 from mindinsight.datavisual.common.enums import PluginNameEnum
+from mindinsight.datavisual.common.log import logger
 from mindinsight.datavisual.data_transform import reservoir
-from mindinsight.conf import settings
-
 
 # Type of the tensor event from external component
 _Tensor = collections.namedtuple('_Tensor', ['wall_time', 'step', 'value', 'filename'])
@@ -29,7 +29,7 @@ TensorEvent = collections.namedtuple(
 
 # config for `EventsData`
 _DEFAULT_STEP_SIZES_PER_TAG = settings.DEFAULT_STEP_SIZES_PER_TAG
-
+_MAX_DELETED_TAGS_SIZE = settings.MAX_TAG_SIZE_PER_EVENTS_DATA * 100
 CONFIG = {
     'max_total_tag_sizes': settings.MAX_TAG_SIZE_PER_EVENTS_DATA,
     'max_tag_sizes_per_plugin':
@@ -60,6 +60,7 @@ class EventsData:
         self._max_step_sizes_per_tag = self._config['max_step_sizes_per_tag']
 
         self._tags = list()
+        self._deleted_tags = set()
         self._reservoir_by_tag = {}
         self._reservoir_mutex_lock = threading.Lock()
 
@@ -82,6 +83,8 @@ class EventsData:
         if tag not in set(self._tags):
             deleted_tag = self._check_tag_out_of_spec(plugin_name)
             if deleted_tag is not None:
+                if tag in self._deleted_tags:
+                    return
                 self.delete_tensor_event(deleted_tag)
 
             self._tags.append(tag)
@@ -114,6 +117,13 @@ class EventsData:
         Args:
             tag (str): The tag name.
         """
+        if len(self._deleted_tags) < _MAX_DELETED_TAGS_SIZE:
+            self._deleted_tags.add(tag)
+        else:
+            logger.warning(
+                'Too many deleted tags, %d upper limit reached, tags updating may not function hereafter',
+                _MAX_DELETED_TAGS_SIZE)
+        logger.warning('%r and all related samples are going to be deleted', tag)
         self._tags.remove(tag)
         for plugin_name, lock in self._tags_by_plugin_mutex_lock.items():
             with lock: