mindspore-mindinsight/tests/st/func/debugger/test_restful_api.py

# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
Function:
    Test query debugger restful api.
Usage:
    pytest tests/st/func/debugger/test_restful_api.py
"""
import os

import pytest

from tests.st.func.debugger.conftest import DEBUGGER_BASE_URL
from tests.st.func.debugger.mock_ms_client import MockDebuggerClient
from tests.st.func.debugger.utils import check_waiting_state, get_request_result, \
    send_and_compare_result


class TestAscendDebugger:
    """Test debugger on Ascend backend."""

    @classmethod
    def setup_class(cls):
        """Setup class."""
        cls._debugger_client = MockDebuggerClient(backend='Ascend')

    @staticmethod
    def _send_terminate_cmd(app_client):
        """Send terminate command to debugger client."""
        url = os.path.join(DEBUGGER_BASE_URL, 'control')
        body_data = {'mode': 'terminate'}
        send_and_compare_result(app_client, url, body_data)

    @staticmethod
    def _create_watchpoint(app_client, condition, expect_id):
        """Create watchpoint."""
        url = 'create_watchpoint'
        body_data = {'condition': condition,
                     'watch_nodes': ['Default/optimizer-Momentum/Parameter[18]_7',
                                     'Default/TransData-op99']}
        res = get_request_result(app_client, url, body_data)
        assert res.get('id') == expect_id

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_before_train_begin(self, app_client):
        """Test retrieve all."""
        url = 'retrieve'
        body_data = {'mode': 'all'}
        expect_file = 'before_train_begin.json'
        send_and_compare_result(app_client, url, body_data, expect_file)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    @pytest.mark.parametrize("body_data, expect_file", [
        ({'mode': 'all'}, 'retrieve_all.json'),
        ({'mode': 'node', 'params': {'name': 'Default'}}, 'retrieve_scope_node.json'),
        ({'mode': 'node', 'params': {'name': 'Default/optimizer-Momentum/Parameter[18]_7'}},
         'retrieve_aggregation_scope_node.json'),
        ({'mode': 'node', 'params': {
            'name': 'Default/TransData-op99',
            'single_node': True}}, 'retrieve_single_node.json'),
        ({'mode': 'watchpoint_hit'}, 'retrieve_empty_watchpoint_hit_list')
    ])
    def test_retrieve_when_train_begin(self, app_client, body_data, expect_file):
        """Test retrieve when train_begin."""
        url = 'retrieve'
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            send_and_compare_result(app_client, url, body_data, expect_file)
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_create_and_delete_watchpoint(self, app_client):
        """Test create and delete watchpoint."""
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            conditions = [
                {'condition': 'MAX_GT', 'param': 1.0},
                {'condition': 'MAX_LT', 'param': -1.0},
                {'condition': 'MIN_GT', 'param': 1e+32},
                {'condition': 'MIN_LT', 'param': -1e+32},
                {'condition': 'MAX_MIN_GT', 'param': 0},
                {'condition': 'MAX_MIN_LT', 'param': 0},
                {'condition': 'MEAN_GT', 'param': 0},
                {'condition': 'MEAN_LT', 'param': 0},
                {'condition': 'INF'},
                {'condition': 'OVERFLOW'},
            ]
            for idx, condition in enumerate(conditions):
                self._create_watchpoint(app_client, condition, idx + 1)
            # delete 4-th watchpoint
            url = 'delete_watchpoint'
            body_data = {'watch_point_id': 4}
            get_request_result(app_client, url, body_data)
            # test watchpoint list
            url = 'retrieve'
            body_data = {'mode': 'watchpoint'}
            expect_file = 'create_and_delete_watchpoint.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_update_watchpoint(self, app_client):
        """Test retrieve when train_begin."""
        watch_point_id = 1
        leaf_node_name = 'Default/optimizer-Momentum/Parameter[18]_7/moments.fc3.bias'
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            condition = {'condition': 'INF'}
            self._create_watchpoint(app_client, condition, watch_point_id)
            # update watchpoint watchpoint list
            url = 'update_watchpoint'
            body_data = {'watch_point_id': watch_point_id,
                         'watch_nodes': [leaf_node_name],
                         'mode': 0}
            get_request_result(app_client, url, body_data)
            # get updated nodes
            url = 'search'
            body_data = {'name': leaf_node_name, 'watch_point_id': watch_point_id}
            expect_file = 'search_unwatched_leaf_node.json'
            send_and_compare_result(app_client, url, body_data, expect_file, method='get')
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_watchpoint_hit(self, app_client):
        """Test retrieve watchpoint hit."""
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            self._create_watchpoint(app_client, condition={'condition': 'INF'}, expect_id=1)
            # send run command to get watchpoint hit
            url = 'control'
            body_data = {'mode': 'continue',
                         'steps': 2}
            res = get_request_result(app_client, url, body_data)
            assert res == {'metadata': {'state': 'running'}}
            # wait for server has received watchpoint hit
            flag = check_waiting_state(app_client)
            assert flag is True
            # check watchpoint hit list
            url = 'retrieve'
            body_data = {'mode': 'watchpoint_hit'}
            expect_file = 'retrieve_watchpoint_hit.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            # check single watchpoint hit
            body_data = {
                'mode': 'watchpoint_hit',
                'params': {
                    'name': 'Default/TransData-op99',
                    'single_node': True,
                    'watch_point_id': 1
                    }
            }
            expect_file = 'retrieve_single_watchpoint_hit.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_retrieve_tensor_value(self, app_client):
        """Test retrieve tensor value."""
        node_name = 'Default/TransData-op99'
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            # prepare tensor value
            url = 'retrieve_tensor_history'
            body_data = {'name': node_name}
            expect_file = 'retrieve_empty_tensor_history.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            # check full tensor history from poll data
            res = get_request_result(
                app_client=app_client, url='poll_data', body_data={'pos': 0}, method='get')
            assert res.get('receive_tensor', {}).get('node_name') == node_name
            expect_file = 'retrieve_full_tensor_history.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            # check tensor value
            url = 'tensors'
            body_data = {
                'name': node_name + ':0',
                'detail': 'data',
                'shape': '[1, 1:3]'
            }
            expect_file = 'retrieve_tensor_value.json'
            send_and_compare_result(app_client, url, body_data, expect_file, method='get')
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_compare_tensor_value(self, app_client):
        """Test compare tensor value."""
        node_name = 'Default/args0'
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            # prepare tensor values
            url = 'control'
            body_data = {'mode': 'continue',
                         'steps': 2}
            get_request_result(app_client, url, body_data)
            flag = check_waiting_state(app_client)
            assert flag is True
            get_request_result(
                app_client=app_client, url='retrieve_tensor_history', body_data={'name': node_name})
            res = get_request_result(
                app_client=app_client, url='poll_data', body_data={'pos': 0}, method='get')
            assert res.get('receive_tensor', {}).get('node_name') == node_name
            # get compare results
            url = 'tensor-comparisons'
            body_data = {
                'name': node_name + ':0',
                'detail': 'data',
                'shape': '[:, :]',
                'tolerance': 1
            }
            expect_file = 'compare_tensors.json'
            send_and_compare_result(app_client, url, body_data, expect_file, method='get')
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    @pytest.mark.parametrize("body_data, expect_file", [
        ({'ascend': True}, 'retrieve_node_by_bfs_ascend.json'),
        ({'name': 'Default/args0', 'ascend': False}, 'retrieve_node_by_bfs.json')
    ])
    def test_retrieve_bfs_node(self, app_client, body_data, expect_file):
        """Test retrieve bfs node."""
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            # prepare tensor values
            url = 'retrieve_node_by_bfs'
            send_and_compare_result(app_client, url, body_data, expect_file, method='get')
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_next_node_on_gpu(self, app_client):
        """Test get next node on GPU."""
        gpu_debugger_client = MockDebuggerClient(backend='GPU')
        with gpu_debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            # send run command to get watchpoint hit
            url = 'control'
            body_data = {'mode': 'continue',
                         'level': 'node',
                         'name': 'Default/TransData-op99'}
            res = get_request_result(app_client, url, body_data)
            assert res == {'metadata': {'state': 'running'}}
            # get metadata
            flag = check_waiting_state(app_client)
            assert flag is True
            url = 'retrieve'
            body_data = {'mode': 'all'}
            expect_file = 'retrieve_next_node_on_gpu.json'
            send_and_compare_result(app_client, url, body_data, expect_file)
            self._send_terminate_cmd(app_client)

    @pytest.mark.level0
    @pytest.mark.env_single
    @pytest.mark.platform_x86_cpu
    @pytest.mark.platform_arm_ascend_training
    @pytest.mark.platform_x86_gpu_training
    @pytest.mark.platform_x86_ascend_training
    def test_pause(self, app_client):
        """Test pause the training."""
        with self._debugger_client.get_thread_instance():
            flag = check_waiting_state(app_client)
            assert flag is True
            # send run command to execute to next node
            url = 'control'
            body_data = {'mode': 'continue',
                         'steps': -1}
            res = get_request_result(app_client, url, body_data)
            assert res == {'metadata': {'state': 'running'}}
            # send pause command
            url = 'control'
            body_data = {'mode': 'pause'}
            res = get_request_result(app_client, url, body_data)
            assert res == {'metadata': {'state': 'waiting'}}
            self._send_terminate_cmd(app_client)