Browse Source

fix the multiprocessing context from fork to forkserver

tags/v1.2.0-rc1
jiangshuqiang jiangshuqiang 4 years ago
parent
commit
5367a64230
1 changed files with 2 additions and 15 deletions
  1. +2
    -15
      mindinsight/utils/computing_resource_mgr.py

+ 2
- 15
mindinsight/utils/computing_resource_mgr.py View File

@@ -13,12 +13,10 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
"""Compute resource manager.""" """Compute resource manager."""
import sys
import fractions import fractions
import math import math
import threading import threading
from concurrent import futures from concurrent import futures
import signal
import multiprocessing import multiprocessing


from mindinsight.utils.log import setup_logger from mindinsight.utils.log import setup_logger
@@ -26,7 +24,7 @@ from mindinsight.utils.constant import GeneralErrors
from mindinsight.utils.exceptions import MindInsightException from mindinsight.utils.exceptions import MindInsightException




_MP_CONTEXT = multiprocessing.get_context(method="fork")
_MP_CONTEXT = multiprocessing.get_context(method="forkserver")
terminating = False terminating = False




@@ -51,18 +49,7 @@ class ComputingResourceManager:
for ind in range(self._executors_cnt) for ind in range(self._executors_cnt)
} }
self._remaining_executors = len(self._executors) self._remaining_executors = len(self._executors)

def initializer():
origin_handler = signal.getsignal(signal.SIGTERM)

def handler(sig, frame):
origin_handler(sig, frame)
sys.exit(0)

signal.signal(signal.SIGTERM, handler)

self._backend = futures.ProcessPoolExecutor(max_workers=max_processes_cnt, mp_context=_MP_CONTEXT,
initializer=initializer)
self._backend = futures.ProcessPoolExecutor(max_workers=max_processes_cnt, mp_context=_MP_CONTEXT)
self.logger = setup_logger("utils", "utils") self.logger = setup_logger("utils", "utils")
self.logger.info("Initialized ComputingResourceManager with executors_cnt=%s, max_processes_cnt=%s.", self.logger.info("Initialized ComputingResourceManager with executors_cnt=%s, max_processes_cnt=%s.",
executors_cnt, max_processes_cnt) executors_cnt, max_processes_cnt)


Loading…
Cancel
Save