commit e7a0496e87b3d7239c0e0b78e7257c104d0c758b Author: gaocongli Date: Fri Mar 27 17:30:35 2020 +0800 initial version diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9028a9a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,89 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +htmlcov +.trash + +.pytest_cache/ +# Distribution / packaging +bin/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Editors/IDEs +.idea/ +*.sublime-* +*.swp +*.save +# test file +.coverage + +.cache + + +# project wide git ignore + +# Compiled artifacts +*.so +*.whl + +# Python backup files +*.pyc + +# Emacs backup files +*~ +*# +.#* + +# Vim file artifacts +.*.sw* + +# Makefile dummy artifacts +.*-dummy + +# log files +*.log + +# code coverage +*.cov + +# Test result xml files +report.xml +*.pprof +results.xml +TESTS*.xml + +# local project settings +.settings +.project +.gradle +.idea + +# tox +.tox/ + +# vscode settings +.vscode + +package-lock.json + +build/lib +build/bdist.* + +output/ +!output/README.md + +third_party/securec/build diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..4eae89e8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +recursive-include mindinsight * +recursive-exclude * .git +recursive-exclude * .gitignore +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] *.swp +recursive-exclude mindinsight/ui * +recursive-include mindinsight/ui/dist * diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..0d896f30 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +MindSpore MindInsight +Copyright 2019-2020 Huawei Technologies Co., Ltd diff --git a/README.md b/README.md new file mode 100644 index 00000000..974ce896 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +MindInsight provides MindSpore with easy-to-use debugging and tuning capabilities. It +enables users to visualize the experiments. The features of MindInsight are as follows. + +- Visualization of training process: + + Provide visualization of training process information, +such as computation graph, training process metrics, etc. + +- Traceability of training result: + + Provide visualization of model parameters information, +such as training data, model accuracy, etc. + + +# Index + +- [More about MindInsight](#more-about-mindinsight) +- [Installation](#installation) +- [QuickStart](#quickstart) +- [Docs](#docs) +- [Community](#community) +- [Contributing](#contributing) +- [Release Notes](#release-notes) +- [License](#license) + +# More about MindInsight + +The architecture diagram of MindInsight is illustrated as follows: + + +![MindInsight Architecture](docs/arch.png) + + +## Summary log file + +The summary log file consists of a series of operation events. Each event contains +the necessary data for visualization. + +MindSpore uses the Callback mechanism to record graph, scalar, image and model +information into summary log file. + +- The scalar and image is recorded by Summary operator. + +- The computation graph is recorded by SummaryRecord after it was compiled. + +- The model parameters is recorded by TrainLineage or EvalLineage. + +MindInsight provides the capability to analyze summary log files and visualize +relative information. + +## Visualization + +MindInsight provides users with a full-process visualized GUI during +AI development, in order to help model developers to improve the model +precision efficiently. + +MindInsight has the following visualization capabilities: + +### Graph visualization + +The GUI of MindInsight displays the structure of neural network, the data flow and control +flow of each operator during the entire training process. + +### Scalar visualization + +The GUI of MindInsight displays the change tendency of a specific scalar during the entire +training process, such as loss value and accuracy rate of each iteration. + +Two scalar curves can be combined and displayed in one chart. + +### Image visualization + +The GUI of MindInsight displays both original images and enhanced images during the entire +training process. + +### Model lineage visualization + +The GUI of MindInsight displays the parameters and metrics of all models, such as the +learning rate, the number of samples and the loss function of each model. + +### Dataset Graph visualization + +The GUI of MindInsight displays the pipeline of dataset processing and augmentation. + +### Dataset Lineage visualization + +The GUI of MindInsight displays the parameters and operations of the dataset processing and augmentation. + +# Installation + +See [Install MindInsight](https://www.mindspore.cn/install/en). + +# QuickStart + +See [guidance](https://www.mindspore.cn/tutorial/en/0.1.0-alpha/advanced_use/visualization_tutorials.html) + +# Docs + +See [API Reference](https://www.mindspore.cn/api/en/master/index.html) + +# Community + +- [MindSpore Slack](https://join.slack.com/t/mindspore/shared_invite/enQtOTcwMTIxMDI3NjM0LTNkMWM2MzI5NjIyZWU5ZWQ5M2EwMTQ5MWNiYzMxOGM4OWFhZjI4M2E5OGI2YTg3ODU1ODE2Njg1MThiNWI3YmQ) - Communication platform for developers. + +# Contributing + +Welcome contributions. See our [Contributor Wiki](https://gitee.com/mindspore/mindspore/blob/master/CONTRIBUTING.md) for more details. + +# Release Notes + +The release notes, see our [RELEASE](RELEASE.md). + +# License + +[Apache License 2.0](LICENSE) diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..e1610167 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,9 @@ +## MindInsight + +# Release 0.1.0-alpha + +* Training process observation + * Provides and displays training process information, including computational graphs and training process indicators. + +* Training result tracing + * Provides functions of tracing and visualizing model training parameter information, including filtering and sorting of training data, model accuracy and training hyperparameters. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..33df57ec --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,6 @@ +# MindInsight Application Scenarios and Security Risks +1. MindInsight is a local tool developed using the HTTP protocol, which is insecure. You are not advised to use it in cloud services or scenarios with security requirements. Otherwise, data may be stolen. +2. The MindInsight source code restricts access from a localhost. If you modify the source code to cancel the localhost binding restriction, data leakage may occur. + +# MindInsight Security Usage Suggestions +- You are advised to create an independent OS user to install and run the MindInsight service. Permissions among OS users are isolated to prevent data theft. In addition, you are advised to set a proper log directory size to prevent log recording exceptions due to insufficient disk space. diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 00000000..e36686f8 --- /dev/null +++ b/build/build.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_BASEDIR=$( + cd "$(dirname "$0")" || exit + pwd +) + +rename_wheel() { + VERSION="$1" + PACKAGE_LIST=$(ls mindinsight-*-any.whl) || exit + for PACKAGE_ORIG in ${PACKAGE_LIST}; do + MINDINSIGHT_VERSION=$(echo "${PACKAGE_ORIG}" | awk -F"-" '{print $2}') + PYTHON_VERSION_NUM=$(echo "${VERSION}" | awk -F"." '{print $1$2}') + PYTHON_VERSION_TAG="cp${PYTHON_VERSION_NUM}" + PYTHON_ABI_TAG="cp${PYTHON_VERSION_NUM}m" + OS_NAME=$(uname | tr '[:upper:]' '[:lower:]') + MACHINE_TAG="${OS_NAME}_$(uname -i)" + PACKAGE_NEW="mindinsight-${MINDINSIGHT_VERSION}-${PYTHON_VERSION_TAG}-${PYTHON_ABI_TAG}-${MACHINE_TAG}.whl" + mv "${PACKAGE_ORIG}" "${PACKAGE_NEW}" + done +} + +build_wheel() { + PROJECT_BASEDIR=$(cd "$(dirname "$SCRIPT_BASEDIR")" || exit; pwd) + cd "${PROJECT_BASEDIR}" || exit + + if [ $# -gt 0 ]; then + if [ "$1" = "clean" ]; then + echo "start cleaning mindinsight" + clean_files + echo "clean mindinsight done" + else + echo "unknown command: $1" + fi + exit + fi + + echo "start building mindinsight" + clean_files + + PYTHON=$(command -v python3 || command -v python) + if [ -z "${PYTHON}" ]; then + echo "Could not find python3 or python command" + exit 1 + fi + PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*') + if [ -z "${PYTHON_VERSION}" ]; then + echo "Could not find Python 3" + exit 1 + fi + + rm -f output + mkdir output + + ${PYTHON} setup.py bdist_wheel + if [ ! -x "dist" ]; then + echo "Build failed" + exit 1 + fi + + mv dist/mindinsight-*-any.whl output/ + + cd output || exit + rename_wheel "${PYTHON_VERSION}" + cd - >/dev/null 2>&1 || exit + + clean_files + + echo "Build success, output directory is: ${PROJECT_BASEDIR}/output" +} + +clean_files() { + rm -rf third_party/build + rm -rf build/lib + rm -rf build/bdist.* + rm -rf mindinsight.egg-info + rm -rf dist +} + +show_usage() { + echo "Build mindinsight" + echo "" + echo "usage: build.sh [-h] [clean]" + echo "" + echo "options:" + echo " -h show usage info" + echo " clean clean build files" +} + +check_opts() { + while getopts ':h' OPT; do + case "$OPT" in + h) + show_usage + exit 0 + ;; + \?) + show_usage + exit 1 + ;; + esac + done +} + +check_opts "$@" + +cd "${SCRIPT_BASEDIR}" || exit +build_wheel "$@" diff --git a/build/scripts/crc32.sh b/build/scripts/crc32.sh new file mode 100755 index 00000000..88b82c6d --- /dev/null +++ b/build/scripts/crc32.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_BASEDIR=$( + cd "$(dirname "$0")" || exit + pwd +) + +THIRD_PARTY_DIR=$(realpath "${SCRIPT_BASEDIR}/../../third_party") +SECUREC_SOURCE_DIR="${THIRD_PARTY_DIR}/securec" + +build_securec() { + CMAKE=$(command -v cmake) + if [ -z "${CMAKE}" ]; then + echo "Could not find cmake command" + exit 1 + fi + + cd "${SECUREC_SOURCE_DIR}" || exit + rm -rf build + mkdir build + cd build || exit + ${CMAKE} .. + make + cd - >/dev/null 2>&1 || exit +} + +build_crc32() { + CPP=$(command -v c++) + if [ -z "${CPP}" ]; then + echo "Could not find c++ command" + exit 1 + fi + + PYTHON=$(command -v python3 || command -v python) + if [ -z "${PYTHON}" ]; then + echo "Could not find python3 or python command" + exit 1 + fi + PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*') + if [ -z "${PYTHON_VERSION}" ]; then + echo "Could not find Python 3" + exit 1 + fi + + DATAVISUAL_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/datavisual") + CRC32_SOURCE_DIR="${DATAVISUAL_DIR}/utils/crc32" + CRC32_OUTPUT_DIR="${DATAVISUAL_DIR}/utils" + CRC32_SO_FILE="crc32$(python3-config --extension-suffix)" + + rm -f "${CRC32_SOURCE_DIR}/${CRC32_SO_FILE}" + rm -f "${CRC32_OUTPUT_DIR}/${CRC32_SO_FILE}" + cd "${CRC32_SOURCE_DIR}" || exit + PYBIND11_INCLUDES=$(${PYTHON} -m pybind11 --includes) + if [ -z "${PYBIND11_INCLUDES}" ]; then + echo "Could not find pybind11 module" + exit 1 + fi + + PYTHON_INCLUDE=$(echo "${PYBIND11_INCLUDES}" | awk '{print $1}' | sed "s/^-I//g") + PYTHON_HEADERS=$(echo "${PYBIND11_INCLUDES}" | awk '{print $2}' | sed "s/^-I//g") + ${CPP} -O2 -O3 -shared -std=c++11 -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 \ + -Wno-maybe-uninitialized -Wno-unused-parameter -Wall -Wl,-z,relro,-z,now,-z,noexecstack \ + -I"${THIRD_PARTY_DIR}" -I"${DATAVISUAL_DIR}/utils" -I"${PYTHON_INCLUDE}" -I"${PYTHON_HEADERS}" \ + -o "${CRC32_SO_FILE}" crc32.cc "${SECUREC_SOURCE_DIR}/build/src/libsecurec.a" + + if [ ! -f "${CRC32_SO_FILE}" ]; then + echo "crc so file does not exist, build failed" + exit 1 + fi + mv "${CRC32_SO_FILE}" "${CRC32_OUTPUT_DIR}" +} + +cd "${SCRIPT_BASEDIR}" || exit +build_securec + +cd "${SCRIPT_BASEDIR}" || exit +build_crc32 diff --git a/build/scripts/ui.sh b/build/scripts/ui.sh new file mode 100755 index 00000000..6b597e29 --- /dev/null +++ b/build/scripts/ui.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_BASEDIR=$( + cd "$(dirname "$0")" || exit + pwd +) + +build_ui() { + NPM=$(command -v npm) + if [ -z "${NPM}" ]; then + echo "Could not find npm command" + exit 1 + fi + + UI_SOURCE_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/ui") + + cd "${UI_SOURCE_DIR}" || exit + rm -rf dist + + ${NPM} config set strict-ssl false + ${NPM} config set unsafe-perm true + ${NPM} config set user 0 + + ${NPM} install + ${NPM} run build + + if [ ! -f "dist/index.html" ]; then + echo "dist does not have file index.html, build failed" + exit 1 + fi + + rm -rf node_modules +} + +cd "${SCRIPT_BASEDIR}" || exit +build_ui diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..4c43a634 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# MindInsight Documentation + +The MindInsight documentation is in the [MindSpore Docs](https://gitee.com/mindspore/docs) repository. diff --git a/docs/arch.png b/docs/arch.png new file mode 100644 index 00000000..128576d0 Binary files /dev/null and b/docs/arch.png differ diff --git a/mindinsight/__init__.py b/mindinsight/__init__.py new file mode 100644 index 00000000..f4a77579 --- /dev/null +++ b/mindinsight/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mindinsight init module.""" + +from mindinsight._version import VERSION + + +__version__ = VERSION +__version_info__ = tuple(VERSION.split('.')) + +__all__ = [ + '__version__', + '__version_info__' +] diff --git a/mindinsight/__main__.py b/mindinsight/__main__.py new file mode 100644 index 00000000..6c93614c --- /dev/null +++ b/mindinsight/__main__.py @@ -0,0 +1,19 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mindinsight main module.""" + +from mindinsight.utils.command import main + +main() diff --git a/mindinsight/_version.py b/mindinsight/_version.py new file mode 100644 index 00000000..8cf7ded2 --- /dev/null +++ b/mindinsight/_version.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mindinsight version module.""" + +VERSION = '0.1.0' diff --git a/mindinsight/backend/__init__.py b/mindinsight/backend/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/backend/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/backend/application.py b/mindinsight/backend/application.py new file mode 100644 index 00000000..5752a8af --- /dev/null +++ b/mindinsight/backend/application.py @@ -0,0 +1,130 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Web application module.""" +import os +from importlib import import_module +from werkzeug.datastructures import Headers +from werkzeug.exceptions import HTTPException + +from flask import Flask +from flask import request +from flask import Response +from flask_cors import CORS + +from mindinsight.conf import settings +from mindinsight.utils.hook import HookUtils +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.common.exceptions import RequestMethodNotAllowed +from mindinsight.datavisual.common import error_handler +from mindinsight.datavisual.utils.tools import find_app_package +from mindinsight.datavisual.utils.tools import get_img_mimetype +from mindinsight.utils.exceptions import MindInsightException + + +def get_security_headers(): + """Get security headers.""" + domain_white_list = [] + for hook in HookUtils.instance().hooks(): + domain_white_list += hook.register_secure_domains() + + content_security_policy = { + 'img-src': ["'self'", 'data:'], + 'style-src': ["'self'", "'unsafe-inline'"], + 'frame-src': ["'self'"] + domain_white_list, + 'frame-ancestors': ["'self'"] + domain_white_list, + 'default-src': ["'self'"], + } + + headers = { + 'X-Frame-Options': 'SAMEORIGIN', + 'X-XSS-Protection': '1; mode=block', + 'X-Content-Type-Options': 'nosniff', + 'Access-Control-Allow-Methods': ', '.join(settings.SUPPORT_REQUEST_METHODS), + 'Content-Security-Policy': '; '.join([ + f"{k} {' '.join(v)}" for k, v in content_security_policy.items() + ]), + 'X-Download-Options': 'noopen', + 'Cache-Control': 'no-store', + 'Pragma': 'no-cache' + } + + return list(headers.items()) + + +SECURITY_HEADERS = get_security_headers() + + +class CustomResponse(Response): + """Define custom response.""" + def __init__(self, response=None, **kwargs): + headers = kwargs.get("headers") + if isinstance(response, bytes): + mimetype = get_img_mimetype(response) + SECURITY_HEADERS.append(('Content-Type', mimetype)) + if headers is None: + headers = Headers(SECURITY_HEADERS) + else: + for header in SECURITY_HEADERS: + headers.add(*header) + kwargs['headers'] = headers + super(CustomResponse, self).__init__(response, **kwargs) + + +def _init_app_module(app): + """ + Init app module. + + Args: + app (Flask): An instance of Flask. + """ + packages = find_app_package() + for package in packages: + try: + app_module = import_module(package) + app_module.init_module(app) + except AttributeError: + logger.debug('[%s].init_module not exists.', package) + + +def before_request(): + """A function to run before each request.""" + if request.method not in settings.SUPPORT_REQUEST_METHODS: + raise RequestMethodNotAllowed() + + +def create_app(): + """Set flask APP config, and start the data manager.""" + static_url_path = "/static" + static_folder_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, 'ui', 'dist', 'static')) + + app = Flask(__name__, static_url_path=static_url_path, static_folder=static_folder_path) + + if settings.ENABLE_CORS: + CORS(app, supports_credentials=True) + + app.before_request(before_request) + + app.register_error_handler(HTTPException, error_handler.handle_http_exception_error) + app.register_error_handler(MindInsightException, error_handler.handle_mindinsight_error) + app.register_error_handler(Exception, error_handler.handle_unknown_error) + + app.response_class = CustomResponse + + _init_app_module(app) + + return app + + +APP = create_app() diff --git a/mindinsight/backend/config/__init__.py b/mindinsight/backend/config/__init__.py new file mode 100644 index 00000000..6d9d0967 --- /dev/null +++ b/mindinsight/backend/config/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Config.""" +import os + +WEB_CONFIG_DIR = os.path.dirname(__file__) diff --git a/mindinsight/backend/config/gunicorn_conf.py b/mindinsight/backend/config/gunicorn_conf.py new file mode 100644 index 00000000..36334474 --- /dev/null +++ b/mindinsight/backend/config/gunicorn_conf.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Config file for gunicorn.""" + +import os +import threading +from importlib import import_module + +import gunicorn + + +gunicorn.SERVER_SOFTWARE = 'unknown' + +worker_class = 'sync' +workers = 1 +threads = min(30, os.cpu_count() * 2 + 1) +worker_connections = 1000 + +timeout = 30 +graceful_timeout = 30 +daemon = True + +captureoutput = True + +# write gunicorn default log to stream, and using mindinsight logger write gunicorn log to file. +accesslog = '-' + + +def on_starting(server): + """Hook function on starting gunicorn process.""" + hook_module = import_module('mindinsight.utils.hook') + for hook in hook_module.HookUtils.instance().hooks(): + threading.Thread(target=hook.on_startup, args=(server.log,)).start() diff --git a/mindinsight/backend/datavisual/__init__.py b/mindinsight/backend/datavisual/__init__.py new file mode 100644 index 00000000..bc325afa --- /dev/null +++ b/mindinsight/backend/datavisual/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Datavisual.""" + +from mindinsight.backend.datavisual.static_resource_api import init_module as static_init_module +from mindinsight.backend.datavisual.task_manager_api import init_module as task_init_module +from mindinsight.backend.datavisual.train_visual_api import init_module as train_init_module + +from mindinsight.conf import settings +from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER + + +def init_module(app): + """ + Interface to init module. + + Args: + app (Flask): An instance of Flask. + + """ + static_init_module(app) + task_init_module(app) + train_init_module(app) + + DATA_MANAGER.start_load_data(reload_interval=int(settings.RELOAD_INTERVAL), + max_threads_count=int(settings.MAX_THREADS_COUNT)) diff --git a/mindinsight/backend/datavisual/static_resource_api.py b/mindinsight/backend/datavisual/static_resource_api.py new file mode 100644 index 00000000..a8d3574a --- /dev/null +++ b/mindinsight/backend/datavisual/static_resource_api.py @@ -0,0 +1,46 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Static resource api.""" +import os +import sys + +from flask import current_app +from flask import send_from_directory +from flask import Blueprint + + +APP_PATH = os.path.realpath(os.path.dirname(sys.argv[0])) +BLUEPRINT = Blueprint("static_resource", __name__) + + +@BLUEPRINT.route("/", methods=["GET"]) +def index(): + """Interface to return static index.html.""" + return send_from_directory(get_index_resource_dir(), "index.html") + + +def get_index_resource_dir(): + """Interface to return index.html resource directory.""" + return os.path.realpath(os.path.join(APP_PATH, current_app.static_folder, os.pardir)) + + +def init_module(app): + """ + Init module entry. + + Args: + app: the application obj. + """ + app.register_blueprint(BLUEPRINT) diff --git a/mindinsight/backend/datavisual/task_manager_api.py b/mindinsight/backend/datavisual/task_manager_api.py new file mode 100644 index 00000000..8bba3246 --- /dev/null +++ b/mindinsight/backend/datavisual/task_manager_api.py @@ -0,0 +1,94 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Task manager api. + +This module provides the interfaces to task manage functions. +""" + +import os + +from flask import Blueprint +from flask import request +from flask import jsonify + +from mindinsight.conf import settings +from mindinsight.datavisual.utils.tools import str_to_bool +from mindinsight.datavisual.utils.tools import get_train_id +from mindinsight.datavisual.processors.train_task_manager import TrainTaskManager +from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher +from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER + + +BLUEPRINT = Blueprint("task_manager", __name__, url_prefix=settings.URL_PREFIX) + + +@BLUEPRINT.route("/datavisual/single-job", methods=["GET"]) +def query_single_train_task(): + """Query single train task""" + plugin_name = request.args.get('plugin_name') + train_id = get_train_id(request) + + processor = TrainTaskManager(DATA_MANAGER) + tasks = processor.get_single_train_task(train_id=train_id, plugin_name=plugin_name) + return jsonify(tasks) + + +@BLUEPRINT.route("/datavisual/plugins", methods=["GET"]) +def query_plugins(): + """Query plugins.""" + train_id = get_train_id(request) + + manual_update = request.args.get('manual_update', default='false') + manual_update = str_to_bool(manual_update, "manual_update") + + processor = TrainTaskManager(DATA_MANAGER) + plugins = processor.get_plugins(train_id, manual_update) + return jsonify(plugins) + + +@BLUEPRINT.route("/datavisual/train-jobs", methods=["GET"]) +def query_train_jobs(): + """Query train jobs.""" + offset = request.args.get("offset", default=0) + limit = request.args.get("limit", default=10) + + summary_watcher = SummaryWatcher() + total, directories = summary_watcher.list_summary_directories_by_pagination( + settings.SUMMARY_BASE_DIR, offset, limit) + + train_jobs = [{ + 'train_id': directory['relative_path'], + 'relative_path': directory['relative_path'], + 'create_time': directory['create_time'].strftime('%Y-%m-%d %H:%M:%S'), + 'update_time': directory['update_time'].strftime('%Y-%m-%d %H:%M:%S'), + } for directory in directories] + + return jsonify({ + 'name': os.path.basename(os.path.realpath(settings.SUMMARY_BASE_DIR)), + 'total': total, + 'train_jobs': train_jobs, + }) + + +def init_module(app): + """ + Init module entry. + + Args: + app: the application obj. + + """ + app.register_blueprint(BLUEPRINT) diff --git a/mindinsight/backend/datavisual/train_visual_api.py b/mindinsight/backend/datavisual/train_visual_api.py new file mode 100644 index 00000000..ababa5c6 --- /dev/null +++ b/mindinsight/backend/datavisual/train_visual_api.py @@ -0,0 +1,156 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Backend interface module. + +This module provides the interfaces to train processors functions. +""" +from flask import Blueprint +from flask import request +from flask import jsonify + +from mindinsight.conf import settings +from mindinsight.datavisual.utils.tools import get_train_id +from mindinsight.datavisual.utils.tools import if_nan_inf_to_none +from mindinsight.datavisual.processors.images_processor import ImageProcessor +from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor +from mindinsight.datavisual.processors.graph_processor import GraphProcessor +from mindinsight.datavisual.data_transform.data_manager import DATA_MANAGER + + +BLUEPRINT = Blueprint("train_visual", __name__, url_prefix=settings.URL_PREFIX) + + +@BLUEPRINT.route("/datavisual/image/metadata", methods=["GET"]) +def image_metadata(): + """ + Interface to fetch metadata about the images for the particular run,tag, and zero-indexed sample. + + Returns: + Response, which contains a list in JSON containing image events, each + one of which is an object containing items wall_time, step, width, + height, and query. + """ + tag = request.args.get("tag") + train_id = get_train_id(request) + + processor = ImageProcessor(DATA_MANAGER) + response = processor.get_metadata_list(train_id, tag) + return jsonify(response) + + +@BLUEPRINT.route("/datavisual/image/single-image", methods=["GET"]) +def single_image(): + """ + Interface to fetch raw image data for a particular image. + + Returns: + Response, which contains a byte string of image. + """ + tag = request.args.get("tag") + step = request.args.get("step") + train_id = get_train_id(request) + + processor = ImageProcessor(DATA_MANAGER) + img_data = processor.get_single_image(train_id, tag, step) + return img_data + + +@BLUEPRINT.route("/datavisual/scalar/metadata", methods=["GET"]) +def scalar_metadata(): + """ + Interface to fetch metadata about the scalars for the particular run and tag. + + Returns: + Response, which contains a list in JSON containing scalar events, each + one of which is an object containing items' wall_time, step and value. + """ + tag = request.args.get("tag") + train_id = request.args.get("train_id") + + processor = ScalarsProcessor(DATA_MANAGER) + response = processor.get_metadata_list(train_id, tag) + + metadatas = response['metadatas'] + for metadata in metadatas: + value = metadata.get("value") + metadata["value"] = if_nan_inf_to_none('scalar_value', value) + + return jsonify(response) + + +@BLUEPRINT.route("/datavisual/graphs/nodes", methods=["GET"]) +def graph_nodes(): + """ + Interface to get graph nodes. + + Returns: + Response, which contains a JSON object. + + """ + name = request.args.get('name', default=None) + node_type = request.args.get('type', default='name_scope') + tag = request.args.get("tag", default=None) + train_id = get_train_id(request) + + graph_process = GraphProcessor(train_id, DATA_MANAGER, tag) + response = graph_process.get_nodes(name=name, node_type=node_type) + return jsonify(response) + + +@BLUEPRINT.route("/datavisual/graphs/nodes/names", methods=["GET"]) +def graph_node_names(): + """ + Interface to query node names. + + Returns: + Response, which contains a JSON object. + """ + search_content = request.args.get("search") + offset = request.args.get("offset", default=0) + limit = request.args.get("limit", default=100) + tag = request.args.get("tag", default=None) + train_id = get_train_id(request) + + graph_process = GraphProcessor(train_id, DATA_MANAGER, tag) + resp = graph_process.search_node_names(search_content, offset, limit) + return jsonify(resp) + + +@BLUEPRINT.route("/datavisual/graphs/single-node", methods=["GET"]) +def graph_search_single_node(): + """ + Interface to search single node. + + Returns: + Response, which contains a JSON object. + """ + name = request.args.get("name") + tag = request.args.get("tag", default=None) + train_id = get_train_id(request) + + graph_process = GraphProcessor(train_id, DATA_MANAGER, tag) + resp = graph_process.search_single_node(name) + return jsonify(resp) + + +def init_module(app): + """ + Init module entry. + + Args: + app (Flask): The application obj. + """ + app.register_blueprint(BLUEPRINT) diff --git a/mindinsight/backend/lineagemgr/__init__.py b/mindinsight/backend/lineagemgr/__init__.py new file mode 100644 index 00000000..e51cfb52 --- /dev/null +++ b/mindinsight/backend/lineagemgr/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +module init file. +""" +from mindinsight.backend.lineagemgr.lineage_api import init_module as init_query_module + + +def init_module(app): + """ + Init module entry. + + Args: + app: Flask. A Flask instance. + + Returns: + + """ + init_query_module(app) diff --git a/mindinsight/backend/lineagemgr/lineage_api.py b/mindinsight/backend/lineagemgr/lineage_api.py new file mode 100644 index 00000000..97108928 --- /dev/null +++ b/mindinsight/backend/lineagemgr/lineage_api.py @@ -0,0 +1,191 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lineage restful api.""" +import json +import os + +from flask import Blueprint, jsonify, request + +from mindinsight.conf import settings +from mindinsight.datavisual.utils.tools import get_train_id +from mindinsight.lineagemgr import filter_summary_lineage, get_summary_lineage +from mindinsight.lineagemgr.common.validator.validate import validate_path +from mindinsight.utils.exceptions import MindInsightException, ParamValueError + +BLUEPRINT = Blueprint("lineage", __name__, url_prefix=settings.URL_PREFIX.rstrip("/")) + + +@BLUEPRINT.route("/models/model_lineage", methods=["POST"]) +def search_model(): + """ + Get model lineage info. + + Get model info by summary base dir return a model lineage information list of dict + contains model's all kinds of param and count of summary log. + + Returns: + str, the model lineage information. + + Raises: + MindInsightException: If method fails to be called. + ParamValueError: If parsing json data search_condition fails. + + Examples: + >>> POST http://xxxx/v1/mindinsight/models/model_lineage + """ + search_condition = request.stream.read() + try: + search_condition = json.loads(search_condition if search_condition else "{}") + except Exception: + raise ParamValueError("Json data parse failed.") + + model_lineage_info = _get_lineage_info( + lineage_type="model", + search_condition=search_condition + ) + + return jsonify(model_lineage_info) + + +@BLUEPRINT.route("/datasets/dataset_lineage", methods=["POST"]) +def get_datasets_lineage(): + """ + Get dataset lineage. + + Returns: + str, the dataset lineage information. + + Raises: + MindInsightException: If method fails to be called. + ParamValueError: If parsing json data search_condition fails. + + Examples: + >>> POST http://xxxx/v1/minddata/datasets/dataset_lineage + """ + search_condition = request.stream.read() + try: + search_condition = json.loads(search_condition if search_condition else "{}") + except Exception: + raise ParamValueError("Json data parse failed.") + + dataset_lineage_info = _get_lineage_info( + lineage_type="dataset", + search_condition=search_condition + ) + + return jsonify(dataset_lineage_info) + + +def _get_lineage_info(lineage_type, search_condition): + """ + Get lineage info for dataset or model. + + Args: + lineage_type (str): Lineage type, 'dataset' or 'model'. + search_condition (dict): Search condition. + + Returns: + dict, lineage info. + + Raises: + MindInsightException: If method fails to be called. + """ + if 'lineage_type' in search_condition: + raise ParamValueError("Lineage type does not need to be assigned in a specific interface.") + if lineage_type == 'dataset': + search_condition.update({'lineage_type': 'dataset'}) + summary_base_dir = str(settings.SUMMARY_BASE_DIR) + try: + lineage_info = filter_summary_lineage( + summary_base_dir, search_condition) + + lineages = lineage_info['object'] + + summary_base_dir = os.path.realpath(summary_base_dir) + length = len(summary_base_dir) + + for lineage in lineages: + summary_dir = lineage['summary_dir'] + summary_dir = os.path.realpath(summary_dir) + if summary_base_dir == summary_dir: + relative_dir = './' + else: + relative_dir = os.path.join(os.curdir, summary_dir[length+1:]) + lineage['summary_dir'] = relative_dir + + except MindInsightException as exception: + raise MindInsightException(exception.error, exception.message, http_code=400) + + return lineage_info + + +@BLUEPRINT.route("/datasets/dataset_graph", methods=["GET"]) +def get_dataset_graph(): + """ + Get dataset graph. + + Returns: + str, the dataset graph information. + + Raises: + MindInsightException: If method fails to be called. + ParamValueError: If summary_dir is invalid. + + Examples: + >>> GET http://xxxx/v1/mindinsight/datasets/dataset_graph?train_id=xxx + """ + + summary_base_dir = str(settings.SUMMARY_BASE_DIR) + summary_dir = get_train_id(request) + if summary_dir.startswith('/'): + validate_path(summary_dir) + elif summary_dir.startswith('./'): + summary_dir = os.path.join(summary_base_dir, summary_dir[2:]) + summary_dir = validate_path(summary_dir) + else: + raise ParamValueError( + "Summary dir should be absolute path or " + "relative path that relate to summary base dir." + ) + try: + dataset_graph = get_summary_lineage( + summary_dir=summary_dir, + keys=['dataset_graph'] + ) + except MindInsightException as exception: + raise MindInsightException(exception.error, exception.message, http_code=400) + + if dataset_graph: + summary_dir_result = dataset_graph.get('summary_dir') + base_dir_len = len(summary_base_dir) + if summary_base_dir == summary_dir_result: + relative_dir = './' + else: + relative_dir = os.path.join( + os.curdir, summary_dir[base_dir_len + 1:] + ) + dataset_graph['summary_dir'] = relative_dir + + return jsonify(dataset_graph) + + +def init_module(app): + """ + Init module entry. + + Args: + app (Flask): The application obj. + """ + app.register_blueprint(BLUEPRINT) diff --git a/mindinsight/backend/run.py b/mindinsight/backend/run.py new file mode 100644 index 00000000..2bd40d51 --- /dev/null +++ b/mindinsight/backend/run.py @@ -0,0 +1,256 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Web service entrance.""" +import os +import stat +import re +import subprocess +import time +import shlex + +from gunicorn.glogging import Logger + +from mindinsight.backend.config import gunicorn_conf +from mindinsight.backend.config import WEB_CONFIG_DIR +from mindinsight.conf import settings +from mindinsight.utils.log import setup_logger + + +MINDBOARD_APP_MODULE = "mindinsight.backend.application:APP" +GUNICORN_LOGGER = "mindinsight.backend.run.GunicornLogger" + +_MIN_PORT = 1 +_MAX_PORT = 65535 + + +def _get_file_size(file_path): + """ + Get the file size. + + Args: + file_path (str): The file path. + + Returns: + int, the file size. If file is not existed, then return 0. + """ + try: + file_size = os.path.getsize(file_path) + except FileNotFoundError: + file_size = 0 + return file_size + + +def _is_match_one(sub_string_list, src_string): + """ + Whether the sub-string in the list can match the source string. + + Args: + sub_string_list (list): The sub-string list. + src_string (str): The source string. + + Returns: + bool, if matched return True, else return False. + """ + for match_info in sub_string_list: + if match_info in src_string: + return True + return False + + +def _check_stat_from_log(log_info): + """ + Determine the service startup status based on the log information. + + Args: + log_info (str): The output log of service startup. + + Returns: + str, the state value that is one of the follows: "unknown", "failed" and "success". + """ + server_state = "unknown" + match_success_info = "Listening at: http://%s:%d" % \ + (settings.HOST, int(settings.PORT)) + common_failed_info_list = [ + "[ERROR] Retrying in 1 second", + "[INFO] Reason: App failed to load", + "[ERROR] Exception in worker process" + ] + re_pattern = "\\[ERROR\\].+%s.+%d" % \ + (settings.HOST, int(settings.PORT)) + + # matched failed output log by fuzzy match + if re.search(re_pattern, log_info) or \ + _is_match_one(common_failed_info_list, log_info): + server_state = "failed" + + if match_success_info in log_info: + server_state = "success" + + return server_state + + +def _get_error_log_path(): + """ + Get gunicorn error log path. + + Returns: + str, the path of error log. + """ + + path = os.path.join(settings.WORKSPACE, 'log/gunicorn/error.log') + errorlog_abspath = os.path.realpath(path) + return errorlog_abspath + + +def _get_access_log_path(): + """Get gunicorn access log path.""" + access_log_path = os.path.join(settings.WORKSPACE, 'log/gunicorn/access.log') + access_log_path = os.path.realpath(access_log_path) + return access_log_path + + +def _check_state_from_log(log_abspath, start_pos=0): + """ + Check the service startup status based on the log file. + + Args: + log_abspath (str): Absolute path of the log file. + start_pos (int): Offset position of the log file. + + Returns: + dict, a dict with "state" and "prompt_message" key. + The value of the "state" key is as follows:"unknown", "failed" and "success". + The value of the "prompt_message" key is a list of prompt messages. + + """ + server_is_start = False + state_result = {"state": "unknown", "prompt_message": []} + prompt_messages = [] + match_start_log = "Starting gunicorn" + with open(log_abspath) as f_log: + f_log.seek(start_pos) + for line in f_log.readlines(): + if match_start_log in line: + if server_is_start: + break + server_is_start = True + continue + if server_is_start: + log_result = _check_stat_from_log(line) + # ignore "unknown" result + if log_result != "unknown": + state_result["state"] = log_result + + if log_result == "failed": + prompt_messages.append(line.strip()) + prompt_messages.append( + "more failed details in log: %s" % log_abspath) + break + state_result["prompt_message"].append( + "service start state: %s" % state_result["state"]) + for prompt_message in prompt_messages: + state_result["prompt_message"].append(prompt_message) + + return state_result + + +def _check_server_start_stat(log_abspath, start_pos=None): + """ + Checking the Server Startup Status. + + Args: + log_abspath (str): The log file path. + start_pos (int): The log file start position. + + Returns: + dict, an dict object that contains the state and prompt_message fields. + The state values are as follows: "unknown", "failed" and "success". + + """ + state_result = {"state": "unknown", "prompt_message": []} + # return unknown when not config gunicorn error log file + if not log_abspath: + return state_result + + log_pos = _get_file_size(log_abspath) if start_pos is None else start_pos + try_cnt = 0 + try_cnt_max = 2 + + while try_cnt < try_cnt_max: + try_cnt += 1 + time.sleep(1) + if _get_file_size(log_abspath) > log_pos: + state_result.update(_check_state_from_log(log_abspath, log_pos)) + break + + return state_result + + +class GunicornLogger(Logger): + """Rewrite gunicorn default logger.""" + + def __init__(self, cfg): + self.access_log = setup_logger('gunicorn', 'access') + self.error_log = setup_logger('gunicorn', 'error') + super(GunicornLogger, self).__init__(cfg) + access_log_path = _get_access_log_path() + error_log_path = _get_error_log_path() + os.chmod(access_log_path, stat.S_IREAD | stat.S_IWRITE) + os.chmod(error_log_path, stat.S_IREAD | stat.S_IWRITE) + + +def start(): + """Start web service.""" + errorlog_abspath = _get_error_log_path() + + gunicorn_conf_file = os.path.join(WEB_CONFIG_DIR, "gunicorn_conf.py") + cmd = "gunicorn " \ + "-b {host}:{port} {app_module} " \ + "-c {conf_file} " \ + "--logger-class {logger_class} " \ + "--access-logformat {log_format}"\ + .format(host=settings.HOST, + port=settings.PORT, + conf_file=gunicorn_conf_file, + app_module=MINDBOARD_APP_MODULE, + logger_class=GUNICORN_LOGGER, + log_format=settings.GUNICORN_ACCESS_FORMAT + ) + + log_size = _get_file_size(errorlog_abspath) + + # start server + process = subprocess.Popen( + shlex.split(cmd), + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + _, stderr = process.communicate() + if stderr: + print(stderr.decode()) + + # wait command success to end when gunicorn running in daemon. + if gunicorn_conf.daemon and process.wait() == 0: + state_result = _check_server_start_stat(errorlog_abspath, log_size) + # print gunicorn start state to stdout + print('Web address: http://{}:{}'.format(settings.HOST, settings.PORT)) + for line in state_result["prompt_message"]: + print(line) + + +if __name__ == '__main__': + start() diff --git a/mindinsight/common/__init__.py b/mindinsight/common/__init__.py new file mode 100644 index 00000000..47b43a6e --- /dev/null +++ b/mindinsight/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/common/hook/__init__.py b/mindinsight/common/hook/__init__.py new file mode 100644 index 00000000..47b43a6e --- /dev/null +++ b/mindinsight/common/hook/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/common/hook/datavisual.py b/mindinsight/common/hook/datavisual.py new file mode 100644 index 00000000..cf3db546 --- /dev/null +++ b/mindinsight/common/hook/datavisual.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Datavisual hook.""" + +import argparse +import os + +from mindinsight.conf import settings +from mindinsight.utils.hook import BaseHook + + +class ReloadIntervalAction(argparse.Action): + """Reload interval action class definition.""" + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Option string for specific argument name. + """ + reload_interval = values + if reload_interval < 0: + parser.error(f'{option_string} should be greater than or equal to 0') + setattr(namespace, self.dest, reload_interval) + + +class SummaryBaseDirAction(argparse.Action): + """Summary base dir action class definition.""" + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Option string for specific argument name. + """ + summary_base_dir = os.path.realpath(values) + setattr(namespace, self.dest, summary_base_dir) + + +class Hook(BaseHook): + """Hook class definition.""" + + def register_startup_arguments(self, parser): + """ + Hook function to register startup arguments. + + Args: + parser (ArgumentParser): Specify parser to which arguments are added. + """ + parser.add_argument( + '--reload-interval', + type=int, + action=ReloadIntervalAction, + help=""" + data reload time(Seconds). It should be greater than 0 or equal to 0. + If it equals 0, load data only once. Default value is %s seconds. + """ % settings.RELOAD_INTERVAL) + + parser.add_argument( + '--summary-base-dir', + type=str, + action=SummaryBaseDirAction, + help=""" + directory where MindInsight will walk through its direct subdirectories + and look for summary files naming with regex 'summary.\\d+' or '\\.pb$'. Any direct + subdirectory containing summary files will turn out to be the summary + file directory. Summary file existing in summary-base-dir indicates that + sumamry-base-dir is one of the summary file directories as well. Default + value is current directory.""") diff --git a/mindinsight/conf/__init__.py b/mindinsight/conf/__init__.py new file mode 100644 index 00000000..c395388c --- /dev/null +++ b/mindinsight/conf/__init__.py @@ -0,0 +1,150 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Conf module.""" + +import os +import json +import types +from importlib import import_module + + +class Settings: + """ + Definition of Settings class. + + Examples: + >>> from mindinsight.conf import settings + >>> print(settings.PORT) + """ + + _prefix = 'MINDINSIGHT_' + _explicit_settings = set() + _default_settings = set() + + def __init__(self): + """Initialization of Settings.""" + self.load_from_defaults() + self.load_from_constants() + self.refresh() + + def refresh(self): + """Refresh settings from config file and environment variables.""" + self.update_from_file() + self.update_from_env() + + def load_from_defaults(self): + """Update settings from defaults module.""" + default_settings = import_module('mindinsight.conf.defaults') + for setting in dir(default_settings): + if setting.isupper(): + setattr(self, setting, getattr(default_settings, setting)) + self._default_settings.add(setting) + + def load_from_constants(self): + """Update settings from constants module""" + constant_settings = import_module('mindinsight.conf.constants') + for setting in dir(constant_settings): + if setting.isupper(): + setattr(self, setting, getattr(constant_settings, setting)) + + def update_from_file(self): + """Update settings from config file.""" + config_path = os.environ.get('MINDINSIGHT_CONFIG', '') + if not config_path: + return + + config_module = None + + # python:full.path.for.config.module + if config_path.startswith('python:'): + config_module = import_module(config_path[len('python:'):]) + + # file:full/path/for/config.py + elif config_path.startswith('file:'): + config_path = config_path[len('file:'):] + module_name = '__mindinsightconfig__' + config_module = types.ModuleType(module_name) + machinery = import_module('importlib.machinery') + loader = machinery.SourceFileLoader(module_name, config_path) + loader.exec_module(config_module) + + if config_module is None: + return + + for setting in dir(config_module): + if setting.isupper() and setting in self._default_settings: + setting_value = getattr(config_module, setting) + setattr(self, setting, setting_value) + self._explicit_settings.add(setting) + + def update_from_env(self): + """Update settings from environment variables.""" + for key, value in os.environ.items(): + if not key.startswith(self._prefix): + continue + + setting = key[len(self._prefix):] + if setting not in self._default_settings: + continue + + setting_value = getattr(self, setting) + if isinstance(setting_value, bool): + value = (value == 'True') + elif isinstance(setting_value, (int, float)): + value = type(setting_value)(value) + elif isinstance(setting_value, (list, dict)): + value = json.loads(value) + + setattr(self, setting, value) + self._explicit_settings.add(setting) + + def config_workspace(self, workspace): + """ + Config workspace value. + + Args: + workspace (str): Path of workspace. + """ + setattr(self, 'WORKSPACE', workspace) + self._explicit_settings.add('WORKSPACE') + + def is_overridden(self, setting_name): + """ + Check if specified setting is overridden. + + Args: + setting_name (str): Setting name to be checked. + + Returns: + bool, indicate whether given setting name is overridden. + """ + return setting_name in self._explicit_settings + + def dump(self): + """ + Dump settings data. + + Returns: + dict, json formatted data of settings. + """ + config = {} + for setting in dir(self): + if setting.isupper(): + config[setting] = getattr(self, setting) + + return config + + +settings = Settings() diff --git a/mindinsight/conf/constants.py b/mindinsight/conf/constants.py new file mode 100644 index 00000000..bd8dd72a --- /dev/null +++ b/mindinsight/conf/constants.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Constants module for mindinsight settings.""" +import logging + +#################################### +# Global default settings. +#################################### +LOG_FORMAT = '[%(levelname)s] MI(%(process)d:%(thread)d,%(processName)s):%(asctime)s ' \ + '[%(filepath)s:%(lineno)d][%(sub_module)s] %(message)s' + +GUNICORN_ACCESS_FORMAT = "'%(h)s <%(r)s> %(s)s %(b)s <%(f)s> <%(a)s> %(D)s'" + +LOG_LEVEL = logging.INFO +# rotating max bytes, default is 50M +LOG_ROTATING_MAXBYTES = 52428800 + +# rotating backup count, default is 30 +LOG_ROTATING_BACKUPCOUNT = 30 + +#################################### +# Web default settings. +#################################### +HOST = '127.0.0.1' + +# Allow to support cross origin resource sharing(CORS) enable. Default is disable. +# If enable CORS, `SUPPORT_REQUEST_METHODS` should enable 'OPTIONS' method. +ENABLE_CORS = False + +SUPPORT_REQUEST_METHODS = {'POST', 'GET', 'PUT', 'DELETE'} + +# url prefix should not end with slash, correct format is /v1/url +URL_PREFIX = '/v1/mindinsight' + +#################################### +# Datavisual default settings. +#################################### +MAX_THREADS_COUNT = 15 + +MAX_TAG_SIZE_PER_EVENTS_DATA = 300 +DEFAULT_STEP_SIZES_PER_TAG = 500 + +MAX_GRAPH_TAG_SIZE = 10 +MAX_IMAGE_STEP_SIZE_PER_TAG = 10 +MAX_SCALAR_STEP_SIZE_PER_TAG = 1000 +MAX_GRAPH_STEP_SIZE_PER_TAG = 1 diff --git a/mindinsight/conf/defaults.py b/mindinsight/conf/defaults.py new file mode 100644 index 00000000..ec3054bd --- /dev/null +++ b/mindinsight/conf/defaults.py @@ -0,0 +1,32 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Defaults module for mindinsight settings.""" +import os + +#################################### +# Global default settings. +#################################### +WORKSPACE = os.path.join(os.environ['HOME'], 'mindinsight') + +#################################### +# Web default settings. +#################################### +PORT = 8080 + +#################################### +# Datavisual default settings. +#################################### +RELOAD_INTERVAL = 3 # Seconds +SUMMARY_BASE_DIR = os.getcwd() diff --git a/mindinsight/datavisual/__init__.py b/mindinsight/datavisual/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/common/__init__.py b/mindinsight/datavisual/common/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/common/enums.py b/mindinsight/datavisual/common/enums.py new file mode 100644 index 00000000..f5347f56 --- /dev/null +++ b/mindinsight/datavisual/common/enums.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Enums.""" + +from enum import Enum + +class BaseEnum(Enum): + + @classmethod + def list_members(cls): + """List all members.""" + return [member.value for member in cls] + + +class DataManagerStatus(BaseEnum): + """Data manager status.""" + INIT = 'INIT' + LOADING = 'LOADING' + DONE = 'DONE' + INVALID = 'INVALID' + + +class PluginNameEnum(BaseEnum): + """Plugin Name Enum.""" + IMAGE = 'image' + SCALAR = 'scalar' + GRAPH = 'graph' diff --git a/mindinsight/datavisual/common/error_handler.py b/mindinsight/datavisual/common/error_handler.py new file mode 100644 index 00000000..393aac36 --- /dev/null +++ b/mindinsight/datavisual/common/error_handler.py @@ -0,0 +1,63 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Handle custom error.""" +from urllib.parse import quote +from werkzeug.exceptions import NotFound +from werkzeug.exceptions import MethodNotAllowed + +from flask import request, jsonify + +from mindinsight.datavisual.common.exceptions import RequestMethodNotAllowed +from mindinsight.datavisual.common.exceptions import RestfulApiNotExist +from mindinsight.datavisual.common.log import restful_logger as logger +from mindinsight.utils.exceptions import UnknownError +from mindinsight.utils.exceptions import FileSystemPermissionError + + +def handle_http_exception_error(ex): + """Handle http exception error.""" + logger.warning('%r %r, detail: %r', request.method, quote(request.path), str(ex)) + if isinstance(ex, NotFound): + error = RestfulApiNotExist() + elif isinstance(ex, MethodNotAllowed): + error = RequestMethodNotAllowed() + else: + logger.exception(ex) + error = UnknownError('System error or http error.') + res_body = {"error_code": error.error_code, "error_msg": error.message} + return jsonify(res_body), error.http_code + + +def handle_mindinsight_error(ex): + """Handle mindinsight error.""" + if int(ex.http_code) < 500: + logger.warning('%r %r detail: %r', request.method, quote(request.path), ex.message) + else: + logger.error('%r %r detail: %r', request.method, quote(request.path), ex.message) + logger.exception(ex) + res_body = dict(error_code=ex.error_code, error_msg=ex.message) + return jsonify(res_body), ex.http_code + + +def handle_unknown_error(ex): + """Handle unknown error.""" + logger.error('%r %r detail: %r', request.method, quote(request.path), str(ex)) + logger.exception(ex) + if isinstance(ex, PermissionError): + error = FileSystemPermissionError('File System Permission Error') + else: + error = UnknownError('System error.') + res_body = dict(error_code=error.error_code, error_msg=error.message) + return jsonify(res_body), error.http_code diff --git a/mindinsight/datavisual/common/exceptions.py b/mindinsight/datavisual/common/exceptions.py new file mode 100644 index 00000000..7b47a482 --- /dev/null +++ b/mindinsight/datavisual/common/exceptions.py @@ -0,0 +1,83 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define custom exception.""" + +from mindinsight.utils.constant import DataVisualErrors +from mindinsight.utils.exceptions import MindInsightException + + +class RestfulApiNotExist(MindInsightException): + """404 not found.""" + def __init__(self): + error_msg = '404 Not Found.' + super(RestfulApiNotExist, self).__init__(DataVisualErrors.RESTFUL_API_NOT_EXIST, + error_msg, + http_code=404) + + +class RequestMethodNotAllowed(MindInsightException): + """Request method not allowed.""" + def __init__(self): + error_msg = '405 Method Not Allowed.' + super(RequestMethodNotAllowed, self).__init__(DataVisualErrors.REQUEST_METHOD_NOT_ALLOWED, + error_msg, + http_code=405) + + +class PathNotDirectoryError(MindInsightException): + """Raised when specified path do not exist.""" + def __init__(self, error_detail): + """Initialize PathNotExistError""" + error_msg = 'Specified path is not a directory. Detail: {}'.format(error_detail) + super(PathNotDirectoryError, self).__init__(DataVisualErrors.PATH_NOT_DIRECTORY_ERROR, + error_msg, + http_code=400) + + +class SummaryLogPathInvalid(MindInsightException): + """No valid log file in the path.""" + def __init__(self): + error_msg = 'No valid summary log file in path' + super(SummaryLogPathInvalid, self).__init__(DataVisualErrors.SUMMARY_LOG_PATH_INVALID, + error_msg, + http_code=400) + + +class CRCFailedError(MindInsightException): + """CRC fail, record corrupted.""" + def __init__(self): + error_msg = 'CRC Failed.' + super(CRCFailedError, self).__init__(DataVisualErrors.CRC_FAILED, + error_msg, + http_code=400) + + +class SummaryLogIsLoading(MindInsightException): + """Data is loading.""" + + def __init__(self, error_detail): + error_msg = "Data is loading. Detail: %s" % error_detail + super(SummaryLogIsLoading, self).__init__(DataVisualErrors.SUMMARY_LOG_IS_LOADING, + error_msg, + http_code=400) + + +class NodeNotInGraphError(MindInsightException): + """Can not find node in graph error.""" + def __init__(self): + error_msg = "Can not find node in graph by given node name." + super(NodeNotInGraphError, self).__init__(DataVisualErrors.NODE_NOT_IN_GRAPH_ERROR, + error_msg, + http_code=400) diff --git a/mindinsight/datavisual/common/log.py b/mindinsight/datavisual/common/log.py new file mode 100644 index 00000000..53fd669b --- /dev/null +++ b/mindinsight/datavisual/common/log.py @@ -0,0 +1,19 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Create a logger.""" +from mindinsight.utils.log import setup_logger + +logger = setup_logger("datavisual", "datavisual") +restful_logger = setup_logger("restful_api", "restful_api") diff --git a/mindinsight/datavisual/common/validation.py b/mindinsight/datavisual/common/validation.py new file mode 100644 index 00000000..30b83913 --- /dev/null +++ b/mindinsight/datavisual/common/validation.py @@ -0,0 +1,102 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define a validation class which contain all check methods of datavisual module.""" +from numbers import Number +from mindinsight.utils.exceptions import ParamValueError +from mindinsight.utils.exceptions import ParamMissError +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.utils.tools import to_int + + +class Validation: + """Validation class, define all check methods.""" + + @classmethod + def check_offset(cls, offset, default_value=0): + """ + Check offset parameter, it must be greater or equal 0. + + Args: + offset (Union[str, int]): Value can be string number or int. + default_value (int): Default value for checked offset. Default: 0. + + Returns: + int, offset. + """ + + if offset is None: + return default_value + offset = to_int(offset, 'offset') + if offset < 0: + raise ParamValueError("'offset' should be greater than or equal to 0.") + return offset + + @classmethod + def check_limit(cls, limit, min_value=1, max_value=1000, default_value=100): + """ + Check limit parameter, it should between min_value and max_value. + + Args: + limit (Union[str, int]): Value can be string number or int. + min_value (int): Limit should greater or equal this value. Default: 1. + max_value (int): Limit should less or equal this value. Default: 1000. + default_value (int): Default value for limit. Default: 100. + + Returns: + int, limit. + """ + + if limit is None: + return default_value + + limit = to_int(limit, 'limit') + if limit < min_value or limit > max_value: + raise ParamValueError("'limit' should in [{}, {}].".format(min_value, max_value)) + return limit + + @classmethod + def check_param_empty(cls, **kwargs): + """ + Check param. + + Args: + kwargs (Any): Check if arg is truthy. + + Raises: + ParamMissError: When param missing. + """ + for key, value in kwargs.items(): + # When value is 0, 0.0 or False, it is not empty. + if isinstance(value, Number): + continue + + if not value: + raise ParamMissError(key) + + @classmethod + def check_plugin_name(cls, plugin_name): + """ + Check plugin name. + + Args: + plugin_name (str): The plugin name. + + Raises: + ParamValueError: When plugin name is not valid. + """ + plugin_name_list = PluginNameEnum.list_members() + if plugin_name not in plugin_name_list: + raise ParamValueError("'plugin_name' only can be one of {}" + "".format(plugin_name_list)) diff --git a/mindinsight/datavisual/data_access/__init__.py b/mindinsight/datavisual/data_access/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/data_access/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/data_access/base_file_system.py b/mindinsight/datavisual/data_access/base_file_system.py new file mode 100644 index 00000000..788bcddf --- /dev/null +++ b/mindinsight/datavisual/data_access/base_file_system.py @@ -0,0 +1,68 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base file system.""" +from abc import ABC, abstractmethod +from collections import namedtuple +StatInfo = namedtuple("Info", ["size", "mtime"]) + + +class BaseFileSystem(ABC): + """Base class for file systems.""" + + @abstractmethod + def list_dir(self, path): + """ + Abstract method for listing directories by path. + + Args: + path (str): Directory path or file path. + """ + + @abstractmethod + def is_dir(self, path): + """ + Abstract method for determining if it is a directory. + + Args: + path (str): Directory path or file path. + """ + + @abstractmethod + def exists(self, path): + """ + Abstract method for determining if it exists. + + Args: + path (str): Directory path or file path. + """ + + @abstractmethod + def file_stat(self, file_path): + """ + Abstract method for getting file stat information. + + Args: + file_path (str): File path. + """ + + @abstractmethod + def join(self, path, *paths): + """ + Abstract method for combining paths. + + Args: + path (str): Directory path. + *paths (str): Path or paths. + """ diff --git a/mindinsight/datavisual/data_access/file_handler.py b/mindinsight/datavisual/data_access/file_handler.py new file mode 100644 index 00000000..48392d16 --- /dev/null +++ b/mindinsight/datavisual/data_access/file_handler.py @@ -0,0 +1,290 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""File handler for file operations.""" +from mindinsight.utils.exceptions import PathNotExistError +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.utils.tools import to_str +from mindinsight.datavisual.data_access.local_file_system import LocalFileSystem + +_DEFAULT_BUFFER_SIZE = 24 * 1024 * 1024 + +# _FILE_SYSTEMS, key: FileProtocolHead, value: FileSystem +_FILE_SYSTEMS = dict() +_FILE_SYSTEMS[""] = LocalFileSystem() + + +class FileHandler: + """File handler.""" + + def __init__(self, file_path, mode='rb'): + """ + Init FileHandler. + + Args: + file_path (str): File path. + mode (Literal['r', 'rb', 'br', 'w', 'wb', 'bw']): It must be + in ['r', 'rb', 'br', 'w', 'wb', 'bw']. + """ + logger.debug("The __init__ method enter, param: file_path=%s" + "mode=%s", file_path, mode) + + if mode not in ('r', 'rb', 'br', 'w', 'wb', 'bw'): + raise ValueError("mode %s is not supported by FileHandler." % mode) + + self._file_path = to_str(file_path) + self._file_system = self.get_file_system(self._file_path) + self._buff_chunk_size = _DEFAULT_BUFFER_SIZE + self._buff = None + self._buff_offset = 0 + self._offset = 0 + self._binary_mode = 'b' in mode + + @staticmethod + def get_file_system(path): + """ + Get file system object from path. + + Args: + path (str): Directory path or file path. + + Returns: + BaseFileSystem, a file system object. + """ + path = to_str(path) + prefix_index = path.find("://") + prefix = path[:prefix_index] if prefix_index >= 0 else "" + file_system = _FILE_SYSTEMS.get(prefix, None) + + if file_system is None: + raise ValueError("No filesystem can be found for prefix %s" % prefix) + return file_system + + @staticmethod + def walk(node, forward=True, onerror=None): + """ + Traverse path for directory and file tree. + + Read from the buffer first.If there is not enough data in the buffer, + data will be read from the file system. + + Args: + node (str): Current path. + forward (bool): If True, it will return the sub-directories and files in the top-level + directory first and then iterate the files in the sub-directories. Default: True. + onerror (Optional[Callable]): If None, it indicates that errors during file traversal + will be ignored. Default: None. + + Yields: + Tuple, (node, sub_dirs, files). + + """ + logger.debug("The walk method enter, param: node=%s, " + "forward=%s, onerror=%s.", node, forward, type(onerror)) + + file_system = FileHandler.get_file_system(node) + node = to_str(node) + dirs = [] + + try: + dirs = file_system.list_dir(node) + except PathNotExistError as err: + if onerror: + onerror(err) + else: + logger.warning("Get dir list error, dir_path=%s error=%s.", node, str(err)) + return + + sub_dirs, files = [], [] + for item in dirs: + full_path = file_system.join(node, to_str(item)) + if file_system.is_dir(full_path): + sub_dirs.append(item) + else: + files.append(item) + + result = (node, sub_dirs, files) + + if forward: + logger.debug("The walk method return, pre result=%s.", result) + yield result + + for subdir in sub_dirs: + joined_subdir = file_system.join(node, to_str(subdir)) + for sub_results in FileHandler.walk(joined_subdir, forward, onerror): + yield sub_results + + if not forward: + logger.debug("The walk method return, post result=%s.", result) + yield result + + def read(self, size=None): + """ + Read bytes from buffer or file by size. + + Args: + size (Union[None, int]): Number of bytes to read, If set None, read the whole file. Default: None. + + Returns: + str, a certain number of bytes. + """ + if size is None: + result = self._file_system.read(self._file_path, self._binary_mode) + self._offset = len(result) + return result + + result = None + if self._buff and len(self._buff) > self._buff_offset: + read_offset = self._buff_offset + size if size is not None else len(self._buff) + result = self._read_buffer_by_offset(read_offset) + if size is not None: + if len(result) == size: + return result + size -= len(result) + + read_size = max(self._buff_chunk_size, size) if size is not None else None + self._buff = self._file_system.read(self._file_path, self._binary_mode, + read_size, self._offset) + self._buff_offset = 0 + + read_offset = size if size is not None else len(self._buff) + chunk = self._read_buffer_by_offset(read_offset) + + result = result + chunk if result else chunk + + return result + + def _read_buffer_by_offset(self, new_buff_offset): + """ + Read buffer by offset. + + Args: + new_buff_offset (int): Ending offset to read. + + Returns: + str, bytes from old offset to new offset. + + """ + old_buff_offset = self._buff_offset + read_size = min(len(self._buff), new_buff_offset) - old_buff_offset + self._offset += read_size + self._buff_offset += read_size + return self._buff[old_buff_offset:old_buff_offset + read_size] + + def reset_offset(self, offset): + """ + Reset offset and buff_offset, clean buff. + + Args: + offset (int): Offset. + + """ + self._offset = offset + self._buff = None + self._buff_offset = 0 + + @staticmethod + def list_dir(path): + """ + List directories by path. + + Args: + path (str): Directory path or file path. + + Returns: + list[str], directories. + """ + file_system = FileHandler.get_file_system(path) + return file_system.list_dir(path) + + @staticmethod + def is_dir(path): + """ + Determine if it is a directory. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it is a directory path, return True. + """ + file_system = FileHandler.get_file_system(path) + return file_system.is_dir(path) + + @staticmethod + def is_file(path): + """ + Determine if it is a file. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it is a file path, return True. + """ + file_system = FileHandler.get_file_system(path) + return file_system.is_file(path) + + @staticmethod + def exists(path): + """ + Determine if it exists. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it exists, return True. + """ + file_system = FileHandler.get_file_system(path) + return file_system.exists(path) + + @staticmethod + def file_stat(file_path): + """ + Get file stat information. + + Args: + file_path (str): File path. + + Returns: + Nametuple, the (size, mtime) of file. + """ + file_system = FileHandler.get_file_system(file_path) + return file_system.file_stat(file_path) + + @staticmethod + def join(path, *paths): + """ + Join paths. + + Args: + path (str): Directory path. + paths (str): Path or paths. + + Returns: + str, the joined path. + """ + file_system = FileHandler.get_file_system(path) + return file_system.join(path, *paths) + + @property + def offset(self): + """Get offset.""" + return self._offset + + @property + def file_path(self): + """Get file path.""" + return self._file_path diff --git a/mindinsight/datavisual/data_access/local_file_system.py b/mindinsight/datavisual/data_access/local_file_system.py new file mode 100644 index 00000000..377019be --- /dev/null +++ b/mindinsight/datavisual/data_access/local_file_system.py @@ -0,0 +1,143 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Local File System.""" +import io +import os + +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.utils.tools import to_str +from mindinsight.datavisual.data_access.base_file_system import BaseFileSystem +from mindinsight.datavisual.data_access.base_file_system import StatInfo +from mindinsight.utils.exceptions import PathNotExistError + + +class LocalFileSystem(BaseFileSystem): + """Local file system.""" + + def list_dir(self, path): + """ + List directories by path. + + Args: + path (str): Directory path or file path. + + Returns: + list[str], directories. + """ + path = to_str(path) + if not self.is_dir(path): + raise exceptions.PathNotDirectoryError("Path is %s." % path) + return os.listdir(path) + + def is_dir(self, path): + """ + Determine if it is a directory. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it is a directory path, return True. + """ + return os.path.isdir(to_str(path)) + + def is_file(self, path): + """ + Determine if it is a file. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it is a file path, return True. + """ + return os.path.isfile(to_str(path)) + + def exists(self, path): + """ + Determine if it exists. + + Args: + path (str): Directory path or file path. + + Returns: + bool, if it exists, return True. + """ + return os.path.exists(to_str(path)) + + def file_stat(self, file_path): + """ + Get file stat information. + + Args: + file_path (str): File path. + + Returns: + Nametuple, the (size, mtime) of file. + """ + try: + file_info = os.stat(to_str(file_path)) + except OSError: + raise PathNotExistError("File %s is not exist." % file_path) + return StatInfo(size=file_info.st_size, mtime=file_info.st_mtime) + + @staticmethod + def read_access(file_path): + """ + Determine if it has read permission. + + Args: + file_path (str): File path. + + Returns: + bool, if it has read permission, return True. + """ + return os.access(to_str(file_path), os.R_OK) + + def join(self, path, *paths): + """ + Join paths. + + Args: + path (str): Directory path. + paths (str): Path or paths. + + Returns: + str, the joined path. + """ + return os.path.join(path, *paths) + + @staticmethod + def read(file_path, binary_mode=False, size=None, offset=None): + """ + Read file. + + Args: + file_path (str): File path. + binary_mode (bool): If true, mode will be 'rb'. Else, 'r'. + size (int): Size of bytes to read. + offset (int): Offset of file to read. + Returns: + bytes, the content read. + """ + mode = "rb" if binary_mode else "r" + encoding = None if binary_mode else "utf8" + with io.open(file_path, mode, encoding=encoding) as file: + if offset is not None: + file.seek(offset) + if size is not None: + return file.read(size) + + return file.read() diff --git a/mindinsight/datavisual/data_transform/__init__.py b/mindinsight/datavisual/data_transform/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/data_transform/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/data_transform/data_loader.py b/mindinsight/datavisual/data_transform/data_loader.py new file mode 100644 index 00000000..736067c3 --- /dev/null +++ b/mindinsight/datavisual/data_transform/data_loader.py @@ -0,0 +1,70 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +DataLoader is an adapter for all other loaders. + +This module can identify what loader should be used to load data. +""" + +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.data_transform.ms_data_loader import MSDataLoader +from mindinsight.datavisual.common import exceptions + + +class DataLoader: + """ + The adapter of all kinds of loaders. + + Args: + summary_dir (str): A directory path. + """ + def __init__(self, summary_dir): + self._summary_dir = summary_dir + self._loader = None + + def load(self): + """Load the data when loader is exist.""" + if self._loader is None: + ms_dataloader = MSDataLoader(self._summary_dir) + loaders = [ms_dataloader] + for loader in loaders: + if loader.filter_valid_files(): + self._loader = loader + break + + if self._loader is None: + logger.warning("No valid files can be loaded, summary_dir: %s.", self._summary_dir) + raise exceptions.SummaryLogPathInvalid() + + self._loader.load() + + def get_events_data(self): + """ + Get events data from log file. + + Returns: + Optional[EventsData], None or events data. + """ + return self._loader.get_events_data() + + def has_valid_files(self): + """ + Check the directory for valid files. + + Returns: + bool, if the directory has valid files, return True. + """ + ms_dataloader = MSDataLoader(self._summary_dir) + return bool(ms_dataloader.filter_valid_files()) diff --git a/mindinsight/datavisual/data_transform/data_manager.py b/mindinsight/datavisual/data_transform/data_manager.py new file mode 100644 index 00000000..f09e7775 --- /dev/null +++ b/mindinsight/datavisual/data_transform/data_manager.py @@ -0,0 +1,514 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Management of all events data. + +This module exists to all loaders. +It can read events data through the DataLoader. + +This module also acts as a thread pool manager. +""" +import threading +import time + +from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED + +from mindinsight.conf import settings +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.common.enums import DataManagerStatus +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform.loader_generators.loader_generator import MAX_DATA_LOADER_SIZE +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.utils.exceptions import MindInsightException +from mindinsight.utils.exceptions import ParamValueError + + +class DataManager: + """ + DataManager manages a pool of loader which help access events data. + + Each loader helps deal the data of the events. + A loader corresponds to an events_data. + The DataManager build a pool including all the data_loader. + The data_loader provides extracting + method to get the information of events. + """ + def __init__(self, loader_generators): + """ + Initialize the pool of loader and the dict of name-to-path. + + Args: + loader_generators (list[LoaderGenerator]): Loader generators help generate loaders. + + self._status: Refer `datavisual.common.enums.DataManagerStatus`. + self._loader_pool: {'loader_id': }. + + """ + self._loader_pool = {} + self._deleted_id_list = [] + self._status = DataManagerStatus.INIT.value + self._status_mutex = threading.Lock() + self._loader_pool_mutex = threading.Lock() + self._max_threads_count = 30 + self._reload_interval = 3 + + self._loader_generators = loader_generators + + def _add_loader(self, loader): + """ + Add a loader to load data. + + Args: + loader (LoaderStruct): A object of `Loader`. + + """ + if len(self._loader_pool) >= MAX_DATA_LOADER_SIZE: + delete_number = len(self._loader_pool) - MAX_DATA_LOADER_SIZE + 1 + sorted_loaders = sorted(self._loader_pool.items(), + key=lambda loader: loader[1].latest_update_time) + for index in range(delete_number): + delete_loader_id = sorted_loaders[index][0] + self._delete_loader(delete_loader_id) + self._loader_pool.update({loader.loader_id: loader}) + + def _delete_loader(self, loader_id): + """ + Delete loader from loader pool by loader id. + + Args: + loader_id (str): ID of loader. + """ + if self._loader_pool.get(loader_id) is not None: + logger.debug("delete loader %s", loader_id) + self._loader_pool.pop(loader_id) + + def _execute_loader(self, loader_id): + """ + Load data form data_loader. + + If there is something wrong by loading, add logs and delete the loader. + + Args: + loader_id (str): An ID for `Loader`. + + """ + try: + with self._loader_pool_mutex: + loader = self._loader_pool.get(loader_id, None) + if loader is None: + logger.debug("Loader %r has been deleted, will not load data.", loader_id) + return + loader.data_loader.load() + except MindInsightException as ex: + logger.warning("Data loader %r load data failed. " + "Delete data_loader. Detail: %s", loader_id, ex) + + with self._loader_pool_mutex: + self._delete_loader(loader_id) + + def start_load_data(self, + reload_interval=settings.RELOAD_INTERVAL, + max_threads_count=MAX_DATA_LOADER_SIZE): + """ + Start threads for loading data. + + Args: + reload_interval (int): Time to reload data once. + max_threads_count (int): Max number of threads of execution. + + """ + logger.info("Start to load data, reload_interval: %s, " + "max_threads_count: %s.", reload_interval, max_threads_count) + DataManager.check_reload_interval(reload_interval) + DataManager.check_max_threads_count(max_threads_count) + + self._reload_interval = reload_interval + self._max_threads_count = max_threads_count + + thread = threading.Thread(target=self._reload_data, + name='start_load_data_thread') + thread.daemon = True + thread.start() + + def _reload_data(self): + """This function periodically loads the data.""" + # Let gunicorn load other modules first. + time.sleep(1) + while True: + self._load_data() + + if not self._reload_interval: + break + time.sleep(self._reload_interval) + + def reload_data(self): + """ + Reload the data once. + + This function needs to be used after `start_load_data` function. + """ + logger.debug("start to reload data") + thread = threading.Thread(target=self._load_data, + name='reload_data_thread') + thread.daemon = False + thread.start() + + def _load_data(self): + """This function will load data once and ignore it if the status is loading.""" + logger.info("Start to load data, reload interval: %r.", self._reload_interval) + with self._status_mutex: + if self.status == DataManagerStatus.LOADING.value: + logger.debug("Current status is %s , will ignore to load data.", self.status) + return + self.status = DataManagerStatus.LOADING.value + + self._generate_loaders() + self._execute_load_data() + + if not self._loader_pool: + self.status = DataManagerStatus.INVALID.value + else: + self.status = DataManagerStatus.DONE.value + + logger.info("Load event data end, status: %r, and loader pool size is %r.", + self.status, len(self._loader_pool)) + + def _generate_loaders(self): + """This function generates the loader from given path.""" + loader_dict = {} + for generator in self._loader_generators: + loader_dict.update(generator.generate_loaders(self._loader_pool)) + + sorted_loaders = sorted(loader_dict.items(), key=lambda loader: loader[1].latest_update_time) + latest_loaders = sorted_loaders[-MAX_DATA_LOADER_SIZE:] + self._deal_loaders(latest_loaders) + + def _deal_loaders(self, latest_loaders): + """ + This function determines which loaders to keep or remove or added. + + It is based on the given dict of loaders. + + Args: + latest_loaders (list[dict]): A list of . + """ + + with self._loader_pool_mutex: + for loader_id, loader in latest_loaders: + if self._loader_pool.get(loader_id, None) is None: + self._add_loader(loader) + continue + + # If this loader was updated manually before, + # its latest_update_time may bigger than update_time in summary. + if self._loader_pool[loader_id].latest_update_time < loader.latest_update_time: + self._update_loader_latest_update_time(loader_id, loader.latest_update_time) + + def _execute_load_data(self): + """Load data through multiple threads.""" + threads_count = self._get_threads_count() + if not threads_count: + logger.info("Can not find any valid train log path to load, loader pool is empty.") + return + + logger.info("Start to execute load data. threads_count: %s.", threads_count) + + with ThreadPoolExecutor(max_workers=threads_count) as executor: + futures = [] + loader_pool = self._get_snapshot_loader_pool() + for loader_id in loader_pool: + future = executor.submit(self._execute_loader, loader_id) + futures.append(future) + wait(futures, return_when=ALL_COMPLETED) + + @staticmethod + def check_reload_interval(reload_interval): + """ + Check reload interval is valid. + + Args: + reload_interval (int): Reload interval >= 0. + """ + if not isinstance(reload_interval, int): + raise ParamValueError("The value of reload interval should be integer.") + + if reload_interval < 0: + raise ParamValueError("The value of reload interval should be >= 0.") + + @staticmethod + def check_max_threads_count(max_threads_count): + """ + Threads count should be a integer, and should > 0. + + Args: + max_threads_count (int), should > 0. + """ + if not isinstance(max_threads_count, int): + raise ParamValueError("The value of max threads count should be integer.") + if max_threads_count <= 0: + raise ParamValueError("The value of max threads count should be > 0.") + + def _get_threads_count(self): + """ + Use the maximum number of threads available. + + Returns: + int, number of threads. + + """ + threads_count = min(self._max_threads_count, len(self._loader_pool)) + + return threads_count + + def get_train_job_by_plugin(self, train_id, plugin_name): + """ + Get a train job by train job id. + + If the given train job does not has the given plugin data, the tag list will be empty. + + Args: + train_id (str): Get train job info by the given id. + plugin_name (str): Get tags by given plugin. + + Returns: + TypedDict('TrainJobEntity', {'id': str, 'name': str, 'tags': List[str]}), + a train job object. + + """ + self._check_status_valid() + self._check_train_job_exist(train_id, self._loader_pool) + + loader = self._get_loader(train_id) + if loader is None: + logger.warning("No valid summary log in train job %s, " + "or it is not in the cache.", train_id) + return None + + name = loader.name + data_loader = loader.data_loader + + tags = [] + try: + events_data = data_loader.get_events_data() + tags = events_data.list_tags_by_plugin(plugin_name) + except KeyError: + logger.debug("Plugin name %r does not exist " + "in train job %r, and set tags to empty list.", plugin_name, name) + except AttributeError: + logger.debug("Train job %r has been deleted or it has not loaded data, " + "and set tags to empty list.", name) + + result = dict(id=train_id, name=name, tags=tags) + return result + + def delete_train_job(self, train_id): + """ + Delete train job with a train id. + + Args: + train_id (str): ID for train job. + + """ + with self._loader_pool_mutex: + self._delete_loader(train_id) + + def list_tensors(self, train_id, tag): + """ + List tensors of the given train job and tag. + + If the tensor can not find by the given tag, will raise exception. + + Args: + train_id (str): ID for train job. + tag (str): The tag name. + + Returns: + NamedTuple, the tuple format is `collections.namedtuple('_Tensor', ['wall_time', 'event_step', 'value'])`. + the value will contain the given tag data. + + """ + self._check_status_valid() + loader_pool = self._get_snapshot_loader_pool() + if not self._is_loader_in_loader_pool(train_id, loader_pool): + raise ParamValueError("Can not find any data in loader pool about the train job.") + + data_loader = loader_pool[train_id].data_loader + events_data = data_loader.get_events_data() + + try: + tensors = events_data.tensors(tag) + except KeyError: + error_msg = "Can not find any data in this train job by given tag." + raise ParamValueError(error_msg) + + return tensors + + def _check_train_job_exist(self, train_id, loader_pool): + """ + Check train job exist, if not exist, will raise exception. + + Args: + train_id (str): The given train job id. + loader_pool (dict[str, LoaderStruct]): Refer to self._loader_pool. + + Raises: + ParamValueError: Can not found train job in data manager. + """ + is_exist = False + if train_id in loader_pool: + return + for generator in self._loader_generators: + if generator.check_train_job_exist(train_id): + is_exist = True + break + if not is_exist: + raise ParamValueError("Can not find the train job in data manager.") + + def _is_loader_in_loader_pool(self, train_id, loader_pool): + """ + Check train job exist, if not exist, return False. Else, return True. + + Args: + train_id (str): The given train job id. + loader_pool (dict): See self._loader_pool. + + Returns: + bool, if loader in loader pool, return True. + """ + if train_id in loader_pool: + return True + return False + + def _get_snapshot_loader_pool(self): + """ + Create a snapshot of data loader pool to avoid concurrent mutation and iteration issues. + + Returns: + dict, a copy of `self._loader_pool`. + """ + with self._loader_pool_mutex: + return dict(self._loader_pool) + + def _check_status_valid(self): + """Check if the status is valid to load data.""" + + if self.status == DataManagerStatus.INIT.value: + raise exceptions.SummaryLogIsLoading("Data is being loaded, " + "current status: %s." % self._status) + + def get_single_train_job(self, train_id, manual_update=False): + """ + Get train job by train ID. + + Args: + train_id (str): Train ID for train job. + manual_update (bool): If manual update, True. + + Returns: + dict, single train job, if can not find any data, will return None. + """ + self._check_status_valid() + self._check_train_job_exist(train_id, self._loader_pool) + + loader = self._get_loader(train_id, manual_update) + if loader is None: + logger.warning("No valid summary log in train job %s, " + "or it is not in the cache.", train_id) + return None + + train_job = loader.to_dict() + train_job.pop('data_loader') + + plugin_data = {} + for plugin_name in PluginNameEnum.list_members(): + job = self.get_train_job_by_plugin(train_id, plugin_name=plugin_name) + if job is None: + plugin_data[plugin_name] = [] + else: + plugin_data[plugin_name] = job['tags'] + + train_job.update({'tag_mapping': plugin_data}) + + return train_job + + def _get_loader(self, train_id, manual_update=False): + """ + Get loader by train id. + + Args: + train_id (str): Train Id. + manual_update (bool): If manual, True. Else False. + + Returns: + LoaderStruct, the loader. + """ + loader = None + is_reload = False + with self._loader_pool_mutex: + if self._is_loader_in_loader_pool(train_id, self._loader_pool): + loader = self._loader_pool.get(train_id) + + if manual_update and loader is None: + for generator in self._loader_generators: + tmp_loader = generator.generate_loader_by_train_id(train_id) + if loader and loader.latest_update_time > tmp_loader.latest_update_time: + continue + loader = tmp_loader + + if loader is None: + return None + + self._add_loader(loader) + is_reload = True + + if manual_update: + self._update_loader_latest_update_time(loader.loader_id) + + if is_reload: + self.reload_data() + + return loader + + def _update_loader_latest_update_time(self, loader_id, latest_update_time=None): + """ + Update loader with latest_update_time. + + Args: + loader_id (str): ID of loader. + latest_update_time (float): Timestamp. + """ + if latest_update_time is None: + latest_update_time = time.time() + self._loader_pool[loader_id].latest_update_time = latest_update_time + + @property + def status(self): + """ + Get the status of data manager. + + Returns: + DataManagerStatus, the status of data manager. + """ + return self._status + + @status.setter + def status(self, status): + """Set data manger status.""" + self._status = status + + +_loader_generators = [DataLoaderGenerator(settings.SUMMARY_BASE_DIR)] +DATA_MANAGER = DataManager(_loader_generators) diff --git a/mindinsight/datavisual/data_transform/events_data.py b/mindinsight/datavisual/data_transform/events_data.py new file mode 100644 index 00000000..86150b8c --- /dev/null +++ b/mindinsight/datavisual/data_transform/events_data.py @@ -0,0 +1,216 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Takes a generator of values, and collects them for a frontend.""" + +import collections +import threading + +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform import reservoir +from mindinsight.conf import settings + + +# Type of the tensor event from external component +_Tensor = collections.namedtuple('_Tensor', ['wall_time', 'step', 'value']) +TensorEvent = collections.namedtuple( + 'TensorEvent', ['wall_time', 'step', 'tag', 'plugin_name', 'value']) + +# config for `EventsData` +_DEFAULT_STEP_SIZES_PER_TAG = settings.DEFAULT_STEP_SIZES_PER_TAG + +CONFIG = { + 'max_total_tag_sizes': settings.MAX_TAG_SIZE_PER_EVENTS_DATA, + 'max_tag_sizes_per_plugin': + { + PluginNameEnum.GRAPH.value: settings.MAX_GRAPH_TAG_SIZE, + }, + 'max_step_sizes_per_tag': + { + PluginNameEnum.SCALAR.value: settings.MAX_SCALAR_STEP_SIZE_PER_TAG, + PluginNameEnum.IMAGE.value: settings.MAX_IMAGE_STEP_SIZE_PER_TAG, + PluginNameEnum.GRAPH.value: settings.MAX_GRAPH_STEP_SIZE_PER_TAG, + } +} + + +class EventsData: + """ + EventsData is an event data manager. + + It manages the log events generated during a training process. + The log event records information such as graph, tag, and tensor. + Data such as tensor can be retrieved based on its tag. + """ + + def __init__(self): + self._config = CONFIG + self._max_step_sizes_per_tag = self._config['max_step_sizes_per_tag'] + + self._tags = list() + self._reservoir_by_tag = {} + self._reservoir_mutex_lock = threading.Lock() + + self._tags_by_plugin = collections.defaultdict(list) + self._tags_by_plugin_mutex_lock = collections.defaultdict(threading.Lock) + + def add_tensor_event(self, tensor_event): + """ + Add a new tensor event to the tensors_data. + + Args: + tensor_event (TensorEvent): Refer to `TensorEvent` object. + """ + if not isinstance(tensor_event, TensorEvent): + raise TypeError('Expect to get data of type `TensorEvent`.') + + tag = tensor_event.tag + plugin_name = tensor_event.plugin_name + + if tag not in set(self._tags): + deleted_tag = self._check_tag_out_of_spec(plugin_name) + if deleted_tag is not None: + self.delete_tensor_event(deleted_tag) + + self._tags.append(tag) + + with self._tags_by_plugin_mutex_lock[plugin_name]: + if tag not in self._tags_by_plugin[plugin_name]: + self._tags_by_plugin[plugin_name].append(tag) + + with self._reservoir_mutex_lock: + if tag not in self._reservoir_by_tag: + reservoir_size = self._get_reservoir_size(tensor_event.plugin_name) + self._reservoir_by_tag[tag] = reservoir.Reservoir(reservoir_size) + + tensor = _Tensor(wall_time=tensor_event.wall_time, + step=tensor_event.step, + value=tensor_event.value) + + if self._is_out_of_order_step(tensor_event.step, tensor_event.tag): + self.purge_reservoir_data(tensor_event.step, self._reservoir_by_tag[tag]) + + self._reservoir_by_tag[tag].add_sample(tensor) + + def delete_tensor_event(self, tag): + """ + This function will delete tensor event by the given tag in memory record. + + Args: + tag (str): The tag name. + """ + self._tags.remove(tag) + for plugin_name, lock in self._tags_by_plugin_mutex_lock.items(): + with lock: + if tag in self._tags_by_plugin[plugin_name]: + self._tags_by_plugin[plugin_name].remove(tag) + break + + with self._reservoir_mutex_lock: + if tag in self._reservoir_by_tag: + self._reservoir_by_tag.pop(tag) + + def list_tags_by_plugin(self, plugin_name): + """ + Return all the tag names of the plugin. + + Args: + plugin_name (str): The Plugin name. + + Returns: + list[str], tags of the plugin. + + Raises: + KeyError: when plugin name could not be found. + """ + if plugin_name not in self._tags_by_plugin: + raise KeyError('Plugin %r could not be found.' % plugin_name) + with self._tags_by_plugin_mutex_lock[plugin_name]: + # Return a snapshot to avoid concurrent mutation and iteration issues. + return list(self._tags_by_plugin[plugin_name]) + + def tensors(self, tag): + """ + Return all tensors of the tag. + + Args: + tag (str): The tag name. + + Returns: + list[_Tensor], the list of tensors to the tag. + """ + if tag not in self._reservoir_by_tag: + raise KeyError('TAG %r could not be found.' % tag) + return self._reservoir_by_tag[tag].samples() + + def _is_out_of_order_step(self, step, tag): + """ + If the current step is smaller than the latest one, it is out-of-order step. + + Args: + step (int): Check if the given step out of order. + tag (str): The checked tensor of the given tag. + + Returns: + bool, boolean value. + """ + if self.tensors(tag): + tensors = self.tensors(tag) + last_step = tensors[-1].step + if step <= last_step: + return True + return False + + @staticmethod + def purge_reservoir_data(start_step, tensor_reservoir): + """ + Purge all tensor event that are out-of-order step after the given start step. + + Args: + start_step (int): Urge start step. All previously seen events with + a greater or equal to step will be purged. + tensor_reservoir (Reservoir): A `Reservoir` object. + + Returns: + int, the number of items removed. + """ + cnt_out_of_order = tensor_reservoir.remove_sample(lambda x: x.step < start_step) + + return cnt_out_of_order + + def _get_reservoir_size(self, plugin_name): + max_step_sizes_per_tag = self._config['max_step_sizes_per_tag'] + return max_step_sizes_per_tag.get(plugin_name, _DEFAULT_STEP_SIZES_PER_TAG) + + def _check_tag_out_of_spec(self, plugin_name): + """ + Check whether the tag is out of specification. + + Args: + plugin_name (str): The given plugin name. + + Returns: + Union[str, None], if out of specification, will return the first tag, else return None. + + """ + tag_specifications = self._config['max_tag_sizes_per_plugin'].get(plugin_name) + if tag_specifications is not None and len(self._tags_by_plugin[plugin_name]) >= tag_specifications: + deleted_tag = self._tags_by_plugin[plugin_name][0] + return deleted_tag + + if len(self._tags) >= self._config['max_total_tag_sizes']: + deleted_tag = self._tags[0] + return deleted_tag + + return None diff --git a/mindinsight/datavisual/data_transform/graph/__init__.py b/mindinsight/datavisual/data_transform/graph/__init__.py new file mode 100644 index 00000000..d0406a09 --- /dev/null +++ b/mindinsight/datavisual/data_transform/graph/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file is used to define the graph.""" + +from .msgraph import MSGraph +from .node import NodeTypeEnum + +__all__ = ['MSGraph', 'NodeTypeEnum'] diff --git a/mindinsight/datavisual/data_transform/graph/graph.py b/mindinsight/datavisual/data_transform/graph/graph.py new file mode 100644 index 00000000..4bdaf139 --- /dev/null +++ b/mindinsight/datavisual/data_transform/graph/graph.py @@ -0,0 +1,455 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This file is used to define the basic graph. +""" +import copy +import time + +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.common import exceptions +from .node import NodeTypeEnum +from .node import Node + + +class EdgeTypeEnum: + """Node edge type enum.""" + control = 'control' + data = 'data' + + +class DataTypeEnum: + """Data type enum.""" + DT_TENSOR = 13 + + +class Graph: + """The `Graph` object is used to describe a graph file.""" + MIN_POLYMERIC_NODE_COUNT = 5 + + def __init__(self): + # Store nodes contain leaf nodes, name scope node, except polymeric nodes + self._normal_nodes = {} + + # Store polymeric nodes. + self._polymeric_nodes = {} + + # Store all nodes resolved from the file. + self._leaf_nodes = {} + + # The format of node groups is {'group_name': {'node_name': }} + self._node_groups = {} + + def exist_node(self, name): + """ + Check node exist in graph. + + Args: + name (str): The node name. + + Returns: + bool, if node is exist will return True. + + """ + if self._normal_nodes.get(name) is None: + return False + return True + + def get_normal_nodes(self, namescope=None): + """ + Get nodes by namescope. + + Args: + namescope (str): A namescope of nodes. + + Returns: + list[dict], a list object contain `Node` object. + + """ + nodes = [] + if namescope is None: + for name, node in self._normal_nodes.items(): + if '/' not in name: + # Get first layer nodes + nodes.append(node.to_dict()) + return nodes + + namescope = namescope + '/' + for name, node in self._normal_nodes.items(): + if name.startswith(namescope) and '/' not in name.split(namescope)[1]: + nodes.append(node.to_dict()) + + return nodes + + def get_polymeric_nodes(self, polymeric_scope): + """ + Get polymeric nodes by polymeric scope. + + Args: + polymeric_scope (str): The polymeric scope name of nodes. + + Returns: + list[dict], a list object contain `Node` object. + """ + nodes = [] + for node in self._polymeric_nodes.values(): + if node.polymeric_scope_name == polymeric_scope: + nodes.append(node.to_dict()) + return nodes + + def search_node_names(self, content, offset, limit): + """ + Search node names by content. + + Args: + content (Union[str, None]): This content can be the key content of the node to search, + if None, will get all node names. + offset (int): An offset for page. Ex, offset is 0, mean current page is 1. + limit (int): An offset for page. Ex, offset is 0, mean current page is 1. + + Returns: + list[str], a list of node names. + """ + all_names = [] + all_names.extend(list(self._normal_nodes.keys())) + all_names.extend(list(self._polymeric_nodes.keys())) + if content is not None: + content = content.lower() + catch_names = [name for name in all_names if content in name.lower()] + else: + catch_names = all_names + catch_names = sorted(catch_names) + real_offset = offset * limit + return catch_names[real_offset:real_offset+limit] + + def search_single_node(self, node_name): + """ + Search node, and return every layer nodes until this node. + + Args: + node_name (str): The name of node. + + Returns: + dict, a dict object, format is : + item_object = {'nodes': [], + 'scope_name': '', + 'children': {}} + """ + if node_name and self._polymeric_nodes.get(node_name) is None \ + and self._normal_nodes.get(node_name) is None: + raise exceptions.NodeNotInGraphError() + + response = {} + nodes = self.get_normal_nodes() + response.update({ + 'nodes': nodes, + 'scope_name': '', + 'children': {} + }) + + names = node_name.split('/') + children = response['children'] + for i in range(1, len(names)+1): + if i == len(names): + polymeric_node = self._polymeric_nodes.get(node_name) + if polymeric_node: + polymeric_scope = polymeric_node.polymeric_scope_name + nodes = self.get_polymeric_nodes(polymeric_scope) + children.update({'nodes': nodes, + 'scope_name': polymeric_scope, + 'children': {}}) + break + + name_scope = '/'.join(names[:i]) + nodes = self.get_normal_nodes(name_scope) + children.update({ + 'nodes': nodes, + 'scope_name': name_scope, + 'children': {} + }) + children = children['children'] + + return response + + def _build_polymeric_nodes(self): + """Build polymeric node.""" + logger.debug("Start to build polymeric nodes") + + self._find_polymeric_nodes() + + group_count_map = {} + for group_name, group in self._node_groups.items(): + name = group_name.split('/')[-1] + count = group_count_map.get(name, 0) + count += 1 + group_count_map[name] = count + polymeric_node_name = group_name + '_{}_[{}]'.format(count, len(group)) + polymeric_node = Node(polymeric_node_name, node_id=polymeric_node_name) + polymeric_node.node_type = NodeTypeEnum.POLYMERIC_SCOPE.value + polymeric_node.name_scope = '/'.join(group_name.split('/')[:-1]) + polymeric_node.subnode_count = len(group) + + for name_tmp, node_tmp in group.items(): + node_tmp.polymeric_scope_name = polymeric_node_name + self._polymeric_nodes.update({name_tmp: node_tmp}) + polymeric_node.update_input(node_tmp.input) + polymeric_node.update_output(node_tmp.output) + + self._normal_nodes.update({polymeric_node_name: polymeric_node}) + + self._update_input_output() + + def _find_polymeric_nodes(self): + """Find polymeric nodes from node groups.""" + node_groups = copy.deepcopy(self._node_groups) + for group_name, group in node_groups.items(): + if len(group) < self.MIN_POLYMERIC_NODE_COUNT: + self._normal_nodes.update(group) + self._node_groups.pop(group_name) + continue + + move_node_names = [] + is_move_group = False + for node_name, group_node in group.items(): + node_list = [] + is_in_group = False + for dst_name in group_node.output: + node_tmp = self._leaf_nodes[dst_name] + node_list.append(node_tmp) + + start = time.time() + run_count = 0 + visit_nodes = {} + while node_list: + # Iterate to find if the output of the node in the group causes a loop + # example: there is a group A, and node_a is a Node in group. + # if there is a loop in node_a, like A/node_a -> B/node_b -> A/node_b + # we will remove the node_a from group A. + node_tmp = node_list[0] + node_list = node_list[1:] + visit_nodes.update({node_tmp.name: True}) + if node_tmp in group.values(): + is_in_group = True + break + for dst_name_tmp in node_tmp.output: + run_count += 1 + node_tmp = self._leaf_nodes[dst_name_tmp] + if visit_nodes.get(dst_name_tmp): + continue + node_list.append(node_tmp) + logger.debug("Find group %s node end, is_in_group: %s, use time: %s, " + "run count: %s.", group_name, is_in_group, + time.time() - start, run_count) + + if is_in_group: + move_node_names.append(node_name) + + if (len(group) - len(move_node_names)) < self.MIN_POLYMERIC_NODE_COUNT: + is_move_group = True + break + + if is_move_group: + self._normal_nodes.update(group) + self._node_groups.pop(group_name) + else: + for name_tmp in move_node_names: + node_tmp = self._node_groups[group_name].pop(name_tmp) + self._normal_nodes.update({name_tmp: node_tmp}) + + def _update_input_output(self): + """We need to update input and output attribute after build polymeric node.""" + for node in self._normal_nodes.values(): + for src_name, input_attr in node.input.items(): + if self._polymeric_nodes.get(src_name): + input_attr['scope'] = NodeTypeEnum.POLYMERIC_SCOPE.value + node.update_input({src_name: input_attr}) + + for dst_name, output_attr in node.output.items(): + if self._polymeric_nodes.get(dst_name): + output_attr['scope'] = NodeTypeEnum.POLYMERIC_SCOPE.value + node.update_output({dst_name: output_attr}) + + for node in self._polymeric_nodes.values(): + for src_name, input_attr in node.input.items(): + if self._polymeric_nodes.get(src_name): + input_attr['scope'] = NodeTypeEnum.POLYMERIC_SCOPE.value + node.update_input({src_name: input_attr}) + + for dst_name, output_attr in node.output.items(): + if self._polymeric_nodes.get(dst_name): + output_attr['scope'] = NodeTypeEnum.POLYMERIC_SCOPE.value + node.update_output({dst_name: output_attr}) + + def _calc_polymeric_input_output(self): + """Calc polymeric input and output after build polymeric node.""" + for name, node in self._normal_nodes.items(): + polymeric_input = {} + for src_name in node.input: + src_node = self._polymeric_nodes.get(src_name) + if node.node_type == NodeTypeEnum.POLYMERIC_SCOPE.value: + src_name = src_name if not src_node else src_node.polymeric_scope_name + output_name = self._calc_dummy_node_name(name, src_name) + polymeric_input.update({output_name: {'edge_type': EdgeTypeEnum.data}}) + continue + + if not src_node: + continue + + if not node.name_scope and src_node.name_scope: + # if current node is in first layer, and the src node is not in + # the first layer, the src node will not be the polymeric input of current node. + continue + + if node.name_scope == src_node.name_scope \ + or node.name_scope.startswith(src_node.name_scope): + polymeric_input.update( + {src_node.polymeric_scope_name: {'edge_type': EdgeTypeEnum.data}}) + + node.update_polymeric_input(polymeric_input) + + polymeric_output = {} + for dst_name in node.output: + dst_node = self._polymeric_nodes.get(dst_name) + + if node.node_type == NodeTypeEnum.POLYMERIC_SCOPE.value: + dst_name = dst_name if not dst_node else dst_node.polymeric_scope_name + output_name = self._calc_dummy_node_name(name, dst_name) + polymeric_output.update({output_name: {'edge_type': EdgeTypeEnum.data}}) + continue + + if not dst_node: + continue + + if not node.name_scope and dst_node.name_scope: + continue + + if node.name_scope == dst_node.name_scope \ + or node.name_scope.startswith(dst_node.name_scope): + polymeric_output.update( + {dst_node.polymeric_scope_name: {'edge_type': EdgeTypeEnum.data}}) + + node.update_polymeric_output(polymeric_output) + + for name, node in self._polymeric_nodes.items(): + polymeric_input = {} + for src_name in node.input: + output_name = self._calc_dummy_node_name(name, src_name) + polymeric_input.update({output_name: {'edge_type': EdgeTypeEnum.data}}) + node.update_polymeric_input(polymeric_input) + + polymeric_output = {} + for dst_name in node.output: + polymeric_output = {} + output_name = self._calc_dummy_node_name(name, dst_name) + polymeric_output.update({output_name: {'edge_type': EdgeTypeEnum.data}}) + node.update_polymeric_output(polymeric_output) + + def _calc_dummy_node_name(self, current_node_name, other_node_name): + """ + Calc dummy node name. + + Args: + current_node_name (str): The name of current node. + other_node_name (str): The target dummy node name. + + Returns: + str, the dummy node name. + """ + name_tmp = other_node_name + if self._polymeric_nodes.get(other_node_name): + name_tmp = self._polymeric_nodes[other_node_name].polymeric_scope_name + name_tmp_list = name_tmp.split('/') + current_name_list = current_node_name.split('/') + index = 0 + min_len = min(len(name_tmp_list), len(current_name_list)) + for i in range(min_len): + index = i + if name_tmp_list[index] != current_name_list[index]: + break + dummy_node_name = '/'.join(name_tmp_list[:index+1]) + return dummy_node_name + + def _build_name_scope_nodes(self): + """Build name scope node by every node name.""" + normal_nodes = dict(self._normal_nodes) + + rename_node_names = {} + for name, node in normal_nodes.items(): + name_list = name.split('/') + for i in range(1, len(name_list)): + name_scope = '/'.join(name_list[:i]) + name_scope_node = self._normal_nodes.get(name_scope) + if name_scope_node is None: + name_scope_node = Node(name_scope, node_id=name_scope) + name_scope_node.node_type = NodeTypeEnum.NAME_SCOPE.value + name_scope_node.name_scope = '/'.join(name_list[:i-1]) + elif name_scope_node.node_type != NodeTypeEnum.NAME_SCOPE.value: + # The name of this node conflicts with namescope, so rename this node + old_name = name_scope_node.name + old_names = name_scope_node.name.split('/') + old_names[-1] = f'({old_names[-1]})' + new_name = '/'.join(old_names) + name_scope_node.name = new_name + self._normal_nodes.pop(old_name) + self._normal_nodes.update({new_name: name_scope_node}) + rename_node_names.update({old_name: new_name}) + + # create new namescope + name_scope_node = Node(name_scope, node_id=name_scope) + name_scope_node.node_type = NodeTypeEnum.NAME_SCOPE.value + name_scope_node.name_scope = '/'.join(name_list[:i-1]) + + # update the input and output of this to namescope node + name_scope_with_slash = name_scope + '/' + for src_name, input_attr in node.input.items(): + if src_name.startswith(name_scope_with_slash): + continue + name_scope_node.update_input({src_name: input_attr}) + + for dst_name, output_attr in node.output.items(): + if dst_name.startswith(name_scope_with_slash): + continue + name_scope_node.update_output({dst_name: output_attr}) + + self._normal_nodes.update({name_scope: name_scope_node}) + + if rename_node_names: + # If existing nodes are renamed, the inputs and outputs of all nodes need to be refreshed + nodes = [] + nodes.extend(self._normal_nodes.values()) + nodes.extend(self._polymeric_nodes.values()) + for node in nodes: + attrs = ['input', 'output', 'polymeric_input', 'polymeric_output'] + for item in attrs: + tmp_dict = dict(getattr(node, item)) + for name, value in tmp_dict.items(): + new_name = rename_node_names.get(name, False) + if new_name: + getattr(node, item).pop(name) + getattr(node, f'update_{item}')({new_name: value}) + + self._calc_subnode_count() + + def _calc_subnode_count(self): + """Calc the sub node count of scope node.""" + name_scope_mapping = {} + for node in self._normal_nodes.values(): + if node.name_scope: + count = name_scope_mapping.get(node.name_scope, 0) + name_scope_mapping[node.name_scope] = count + 1 + + for name_scope, count in name_scope_mapping.items(): + node = self._normal_nodes[name_scope] + node.subnode_count = count diff --git a/mindinsight/datavisual/data_transform/graph/msgraph.py b/mindinsight/datavisual/data_transform/graph/msgraph.py new file mode 100644 index 00000000..607002f5 --- /dev/null +++ b/mindinsight/datavisual/data_transform/graph/msgraph.py @@ -0,0 +1,274 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file is used to define the MindSpore graph.""" +import re +import copy + +from mindinsight.datavisual.common.log import logger +from .node import Node +from .node import NodeTypeEnum +from .graph import Graph +from .graph import EdgeTypeEnum +from .graph import DataTypeEnum + + +class MSGraph(Graph): + """The object describes the MindSpore graph, and it is defined in the anf_if proto file.""" + + def build_graph(self, graph_proto): + """ + Build graph by graph proto which refer to `anf_ir_pb2.GraphProto`, and set status to loading. + + Args: + graph_proto (anf_ir_pb2.GraphProto): Refer to `anf_ir_pb2.GraphProto`. + """ + logger.info("Start to build graph.") + + self._build_leaf_nodes(graph_proto) + self._build_polymeric_nodes() + self._build_name_scope_nodes() + self._calc_polymeric_input_output() + logger.info("Build graph end, normal node count: %s, polymeric node " + "count: %s.", len(self._normal_nodes), len(self._polymeric_nodes)) + + def _build_leaf_nodes(self, graph_proto): + """ + Build leaf node from graph proto. + + Left node will contain operation node, parameter node, const node. + + Args: + graph_proto (anf_ir_pb2.model_proto.graph): Refer to anf_ir_pb2.model_proto.graph. + """ + logger.info("Start to build leaf nodes.") + leaf_node_id_map_name = {} + const_nodes_map = {} + + for node_def in graph_proto.node: + node = self._parse_graph_proto_node(node_def) + leaf_node_id_map_name.update({node.node_id: node.name}) + + for parameter in graph_proto.parameters: + node = self._parse_graph_proto_parameter(parameter) + const_nodes_map.update({node.name: node}) + + for i, const in enumerate(graph_proto.const_vals): + node_id = 'const_{}'.format(i) + node = self._parse_graph_proto_const(const, node_id) + const_nodes_map.update({const.key: node}) + + self._calc_input(leaf_node_id_map_name, graph_proto, const_nodes_map) + self._calc_output() + + logger.info("Build leaf nodes end, normal nodes count: %s, group count: %s, " + "left node count: %s.", len(self._normal_nodes), len(self._node_groups), + len(self._leaf_nodes)) + + def _calc_input(self, leaf_node_id_map_name, graph_proto, const_nodes_map): + """ + Calc input for every leaf node. + + Args: + leaf_node_id_map_name (dict[str, str]): Format is {'node_id': 'node_name'}. + graph_proto (anf_ir_pb2.model_proto.graph): See anf_ir_pb2.model_proto.graph. + const_nodes_map (dict[str, Node]): Format is {'node name': }. + """ + logger.debug("Start to calc input.") + for node_def in graph_proto.node: + node_name = leaf_node_id_map_name[node_def.name] + node = self._leaf_nodes[node_name] + for input_def in node_def.input: + edge_type = EdgeTypeEnum.data + if input_def.type == "CONTROL_EDGE": + edge_type = EdgeTypeEnum.control + + if const_nodes_map.get(input_def.name): + const_node = copy.deepcopy(const_nodes_map[input_def.name]) + src_name = '{}/{}'.format(node.name_scope, input_def.name) + if not self._normal_nodes.get(src_name): + const_node.name = src_name + const_node.name_scope = node.name_scope + self._normal_nodes.update({src_name: const_node}) + self._leaf_nodes.update({src_name: const_node}) + src_node = self._leaf_nodes.get(src_name) + else: + src_name = leaf_node_id_map_name.get(input_def.name) + if not src_name: + logger.warning("The input_def name '%s' in node '%s' is invalid, " + "will be ignore.", input_def.name, node_name) + continue + + src_node = self._leaf_nodes.get(src_name) + if src_node is None: + logger.warning("The input '%s' in node '%s' is not in " + "leaf nodes.", src_name, node_name) + continue + + input_item = { + src_name: { + "shape": src_node.shape, + "edge_type": edge_type, + "scope": NodeTypeEnum.NAME_SCOPE.value + } + } + node.update_input(input_item) + + if self._normal_nodes.get(node_name): + self._normal_nodes[node_name] = node + else: + group_name = self._create_group_name(node.name_scope, node.node_type, node.name) + self._node_groups[group_name][node.name] = node + + def _calc_output(self): + """Calc output of every node.""" + logger.debug("Start to calc output.") + + for name, node in self._leaf_nodes.items(): + if node.node_type == NodeTypeEnum.CONST.value: + continue + for src_name, input_attr in node.input.items(): + src_node = self._leaf_nodes[src_name] + if src_node.node_type == NodeTypeEnum.CONST.value: + continue + + if self._normal_nodes.get(src_name): + self._normal_nodes[src_name].update_output({name: input_attr}) + else: + group_name = self._create_group_name(src_node.name_scope, + src_node.node_type, src_node.name) + self._node_groups[group_name][src_name].update_output({name: input_attr}) + + def _parse_graph_proto_node(self, node_def): + """ + Parse `anf_ir_pb2.model_proto.graph.node_def`, and create a a node. + + Args: + node_def (anf_ir_pb2.model_proto.graph.node_def): Refer to anf_ir_pb2.model_proto.graph.node_def. + + Returns: + Node, a `Node` object. + """ + node_name = '/'.join([node_def.scope, node_def.op_type])+node_def.name + node = Node(name=node_name, node_id=node_def.name) + node.node_type = node_def.op_type + logger.debug("Foreach graph proto nodes, node id: %s, node name: %s, node def name: %s, " + "input count: %s", node.node_id, node.name, node_def.name, len(node_def.input)) + + for attr in node_def.attribute: + node.update_attr({attr.name: str(attr.value)}) + + node.output_i = node_def.output_i + node.name_scope = node_def.scope + + output_type = node_def.output_type + shape = self._parse_type_proto(output_type) + node.shape = shape + + self._leaf_nodes.update({node.name: node}) + group_name = self._create_group_name(node.name_scope, node.node_type, node.name) + if group_name is not None: + node_dict = self._node_groups.get(group_name, {}) + node_dict.update({node.name: node}) + self._node_groups.update({group_name: node_dict}) + else: + self._normal_nodes.update({node.name: node}) + + return node + + def _parse_graph_proto_parameter(self, parameter): + """ + Parse anf_ir_pb2.model_proto.graph.parameter, and create a parameter node. + + Args: + parameter (anf_ir_pb2.model_proto.graph.parameter): Refer to anf_ir_pb2.model_proto.graph.parameter. + + Returns: + Node, a `Node` object. + """ + node = Node(name=parameter.name, node_id=parameter.name) + node.node_type = NodeTypeEnum.PARAMETER.value + node.shape = self._parse_type_proto(parameter.type) + logger.debug("Foreach graph proto parameters, node id: %s, node name: %s, " + "node def name: %s", node.node_id, node.name, parameter.name) + return node + + def _parse_graph_proto_const(self, const, const_node_id): + """ + Parse anf_ir_pb2.model_proto.graph.const, and create a const node. + + Args: + const (anf_ir_pb2.model_proto.graph.const): Refer to anf_ir_pb2.model_proto.graph.const + const_node_id (str): The id of the new const node, it should be unique in graph. + + Returns: + Node, a `Node` object. + """ + node = Node(name=const.key, node_id=const_node_id) + node.node_type = NodeTypeEnum.CONST.value + node.update_attr({const.key: str(const.value)}) + if const.value.dtype == DataTypeEnum.DT_TENSOR: + shape = [] + for dim in const.value.tensor_val.dims: + shape.append(dim) + node.shape = shape + return node + + def _parse_type_proto(self, type_proto): + """ + Parse proto's `message TypeProto` to get shape information. + + Args: + type_proto (anf_ir_pb2.TypeProto): Refer to anf_ir_pb2.TypeProto. + + Returns: + list, a list of shape. + """ + shapes = [] + if type_proto.HasField('tensor_type'): + tensor_type = type_proto.tensor_type + tensor_shape_proto = tensor_type.shape + for dim in tensor_shape_proto.dim: + shapes.append(dim.size) + if type_proto.HasField('sequence_type'): + for elem_type in type_proto.sequence_type.elem_types: + shapes.append(self._parse_type_proto(elem_type)) + return shapes + + def _create_group_name(self, name_scope, node_type, node_name): + """ + Create group name by node name, name scope, node type. + + Only nodes that conform to the rules are aggregated. + + Args: + name_scope (str): The node name scope. + node_type (str): The node type. + node_name (str): The node name. + + Returns: + Optional[str], if match the rules will return a group name, else return None. + """ + group_types = ['Reshape', 'Variable'] + pattern_names = r'.*?/Cast-op\d+' + + if node_type in group_types: + group_name = name_scope + '/' + node_type if name_scope else node_type + return group_name + + if node_type == 'FrameworkOp' and re.search(pattern_names, node_name): + group_name = name_scope + '/' + 'Cast-op' if name_scope else 'Cast-op' + return group_name + + return None diff --git a/mindinsight/datavisual/data_transform/graph/node.py b/mindinsight/datavisual/data_transform/graph/node.py new file mode 100644 index 00000000..2923a2c8 --- /dev/null +++ b/mindinsight/datavisual/data_transform/graph/node.py @@ -0,0 +1,211 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This file is used to define the node of graph and associated base types. +""" +from enum import Enum + +class NodeTypeEnum(Enum): + """Node type enum. The following types are new to our custom.""" + NAME_SCOPE = 'name_scope' + POLYMERIC_SCOPE = 'polymeric_scope' + PARAMETER = 'Parameter' + CONST = 'Const' + + +class Node: + """ + Define a node object. + + Args: + name (str): Name of new node. + node_id (str): The id of this node, and node id is unique in graph. + """ + + def __init__(self, name, node_id): + self._node_id = node_id + self._name = name + self._type = "" + self._attr = dict() + self._input = dict() + self._output_i = -1 + self._output = {} + self._polymeric_input = {} + self._polymeric_output = {} + self._polymeric_scope_name = "" + self._subnode_count = 0 + self._name_scope = "" + self.shape = [] + + def to_dict(self): + """Converts the node object to dictionary format.""" + return { + 'name': self._name, + 'type': self._type, + 'attr': self._attr, + 'input': self._input, + 'output_i': self._output_i, + 'output': self._output, + 'polymeric_input': self._polymeric_input, + 'polymeric_output': self._polymeric_output, + 'subnode_count': self._subnode_count, + 'polymeric_scope_name': self._polymeric_scope_name + } + + @property + def node_id(self): + """The id of this node, and id is unique in graph.""" + return self._node_id + + @property + def name(self): + """Get node name.""" + return self._name + + @name.setter + def name(self, name): + """Set node name.""" + self._name = name + + @property + def node_type(self): + """Get node type.""" + return self._type + + @node_type.setter + def node_type(self, node_type): + """Set node type.""" + self._type = node_type + + @property + def attr(self): + """Get node attr.""" + return self._attr + + def update_attr(self, attr_dict): + """ + Update node attr. + + Args: + attr_dict (dict[str, str]): Format is {'': ''}. + """ + self._attr.update(attr_dict) + + @property + def input(self): + """ + Get all input of current node. + + Returns: + dict[str, dict], format is {'': {'shape': [], 'edge_type', 'scope'}}. + """ + return self._input + + def update_input(self, input_dict): + """ + Update input. + + Args: + input_dict (dict[str, dict]): Format is {'': {'shape': [], 'edge_type', 'scope'}}. + """ + self._input.update(input_dict) + + @property + def output_i(self): + """The memory address of this node when it is in run time.""" + return self._output_i + + @output_i.setter + def output_i(self, output_i): + """Set memory address.""" + self._output_i = output_i + + @property + def polymeric_input(self): + """ + The polymeric input is the input of the polymeric nodes. + + Returns: + dict[str, dict], format is {'': {'edge_type': ''}}. + """ + return self._polymeric_input + + def update_polymeric_input(self, polymeric_input): + """The polymeric input is the input of the polymeric nodes.""" + self._polymeric_input.update(polymeric_input) + + @property + def output(self): + """The output node of this node.""" + return self._output + + def update_output(self, output): + """ + Update output node. + + Args: + output (dict[str, TypedDict('NodeType', {'type': str})]): Format + is {"": {"type": ""}}. + """ + self._output.update(output) + + @property + def polymeric_output(self): + """Get polymeric output.""" + return self._polymeric_output + + def update_polymeric_output(self, polymeric_output): + """ + Update polymeric output. + + Args: + polymeric_output (dict[str, dict): Format is {dst_node.polymeric_scope_name: + {'edge_type': EdgeTypeEnum.data}}). + + """ + self._polymeric_output.update(polymeric_output) + + @property + def polymeric_scope_name(self): + """Get polymeric scope name.""" + return self._polymeric_scope_name + + @polymeric_scope_name.setter + def polymeric_scope_name(self, name): + """Set polymeric scope name.""" + self._polymeric_scope_name = name + + @property + def subnode_count(self): + """The sub node count of this node, if this node is a scope node, this count will not be zero.""" + return self._subnode_count + + @subnode_count.setter + def subnode_count(self, count): + """Set sub node count.""" + self._subnode_count = count + + @property + def name_scope(self): + """Get name scope of this node.""" + return self._name_scope + + @name_scope.setter + def name_scope(self, name_scope): + """Set name scope.""" + self._name_scope = name_scope + + def __str__(self): + return f'' diff --git a/mindinsight/datavisual/data_transform/loader_generators/__init__.py b/mindinsight/datavisual/data_transform/loader_generators/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/mindinsight/datavisual/data_transform/loader_generators/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/data_transform/loader_generators/data_loader_generator.py b/mindinsight/datavisual/data_transform/loader_generators/data_loader_generator.py new file mode 100644 index 00000000..042f282b --- /dev/null +++ b/mindinsight/datavisual/data_transform/loader_generators/data_loader_generator.py @@ -0,0 +1,246 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Data Loader Generator. + +This module generate loaders from summary logs. +""" +import os +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.data_access.file_handler import FileHandler +from mindinsight.datavisual.data_transform.data_loader import DataLoader +from mindinsight.datavisual.data_transform.loader_generators.loader_generator import MAX_DATA_LOADER_SIZE +from mindinsight.datavisual.data_transform.loader_generators.loader_struct import LoaderStruct +from mindinsight.datavisual.data_transform.loader_generators.loader_generator import LoaderGenerator +from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher +from mindinsight.utils.exceptions import ParamValueError + + +class DataLoaderGenerator(LoaderGenerator): + """ + DataLoaderGenerator generate a loader_dict of loader from summary logs. + + Each loader helps deal the data of the events. + It helps DataManager to generate loaders. + """ + def __init__(self, summary_path): + """ + Init DataLoaderGenerator. + + Args: + summary_path (str): A directory path, e.g. '/data/ImageNet/'. + """ + self._summary_path = self._check_and_normalize_summary_path(summary_path) + self._summary_watcher = SummaryWatcher() + + def _check_and_normalize_summary_path(self, summary_path): + """ + Check and normalize summary path. + + Args: + summary_path (str): A directory path, e.g. '/data/ImageNet/'. + + Returns: + str, normalized summary path. + + """ + if summary_path is None: + logger.warning("Summary path is None. It will not init data loader generator.") + raise ParamValueError("Summary path is None.") + + summary_path = os.path.realpath(summary_path) + + return summary_path + + def generate_loaders(self, loader_pool): + """ + Generate loader from summary path, if summary path is empty, will return empty list. + + Args: + loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager. + + Returns: + dict[str, LoaderStruct], a dict of `Loader`. + """ + loader_dict = {} + + if not FileHandler.exists(self._summary_path): + logger.warning("Summary path does not exist. It will not start loading events data. " + "Current path is %r.", self._summary_path) + return loader_dict + + dir_map_mtime_dict = {} + min_modify_time = None + summaries_info = self._summary_watcher.list_summary_directories(self._summary_path) + + for item in summaries_info: + relative_path = item.get("relative_path") + current_dir = FileHandler.join(self._summary_path, relative_path) + dataloader = DataLoader(current_dir) + + if not dataloader.has_valid_files(): + logger.debug("Can not find valid train log file in folder %s , " + "will ignore.", relative_path) + continue + + modify_time = item.get("update_time").timestamp() + + # if loader exists in loader pool and newer time, update its time + loader_id = self._generate_loader_id(relative_path) + loader = loader_pool.get(loader_id) + if loader is not None and loader.latest_update_time > modify_time: + modify_time = loader.latest_update_time + + if not min_modify_time: + # The first load, init min modify time + min_modify_time = modify_time + + # We need to find `MAX_DATA_LOADER_SIZE` newly modified folders. + if len(dir_map_mtime_dict) < MAX_DATA_LOADER_SIZE: + if modify_time < min_modify_time: + min_modify_time = modify_time + dir_map_mtime_dict.update({relative_path: modify_time}) + + else: + if modify_time >= min_modify_time: + dir_map_mtime_dict.update({relative_path: modify_time}) + + sorted_dir_tuple = sorted(dir_map_mtime_dict.items(), + key=lambda d: d[1])[-MAX_DATA_LOADER_SIZE:] + + for relative_path, modify_time in sorted_dir_tuple: + loader_id = self._generate_loader_id(relative_path) + loader = self._generate_loader_by_relative_path(relative_path) + loader_dict.update({loader_id: loader}) + + return loader_dict + + def _generate_loader_by_relative_path(self, relative_path): + """ + Generate loader by relative path. + + Args: + relative_path (str): Relative path of a summary directory, e.g. './log1'. + + Returns: + dict[str, LoaderStruct], a dict of `Loader`. + """ + current_dir = os.path.realpath(FileHandler.join(self._summary_path, relative_path)) + data_loader = DataLoader(current_dir) + loader_id = self._generate_loader_id(relative_path) + loader = LoaderStruct(loader_id=loader_id, + name=self._generate_loader_name(relative_path), + path=current_dir, + latest_update_time=FileHandler.file_stat(current_dir).mtime, + data_loader=data_loader) + return loader + + def _generate_loader_id(self, relative_path): + """ + Generate loader id from relative path. + + Args: + relative_path (str): Relative path of a summary directory, e.g. './log1'. + + Returns: + str, loader_id for `Loader`. + + """ + loader_id = relative_path + return loader_id + + def _generate_loader_name(self, relative_path): + """ + Generate loader name from relative path. + + Args: + relative_path (str): Relative path of a summary directory, e.g. './log1'. + + Returns: + str, loader_name for `Loader`. + + """ + loader_name = relative_path + return loader_name + + def _get_relative_path_from_train_id(self, train_id): + """ + Get relative from train_id. + + Args: + train_id (str): Train ID of a summary directory, e.g. './log1'. + + Returns: + str, relative path of `Loader`. + + """ + relative_path = train_id + + return relative_path + + def check_train_job_exist(self, train_id): + """ + Check if train job exists. + + Args: + train_id (str): Train ID of a summary directory, e.g. './log1'. + + Returns: + bool, if train job exists, return True. + + """ + if not self._is_train_id_valid(train_id): + return False + + relative_path = self._get_relative_path_from_train_id(train_id) + if self._summary_watcher.is_summary_directory(self._summary_path, relative_path): + return True + + return False + + def _is_train_id_valid(self, train_id): + """ + Check if train_id is valid. + + Args: + train_id (str): Train ID of a summary directory, e.g. './log1'. + + Returns: + bool, if train id is valid, return True. + + """ + if not train_id.startswith('./'): + logger.warning("The train_id does not start with './'.") + return False + if len(train_id.split("/")) > 2: + logger.warning("The train_id contains multiple '/'.") + return False + return True + + def generate_loader_by_train_id(self, train_id): + """ + Generate loader by train_id. + + Args: + train_id (str): Train ID of a summary directory, e.g. './log1'. + + Returns: + dict[str, LoaderStruct], a dict of `Loader`. + + """ + relative_path = self._get_relative_path_from_train_id(train_id) + loader = self._generate_loader_by_relative_path(relative_path) + + return loader diff --git a/mindinsight/datavisual/data_transform/loader_generators/loader_generator.py b/mindinsight/datavisual/data_transform/loader_generators/loader_generator.py new file mode 100644 index 00000000..44e7210e --- /dev/null +++ b/mindinsight/datavisual/data_transform/loader_generators/loader_generator.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base loader generator.""" +from abc import abstractmethod + +MAX_DATA_LOADER_SIZE = 15 + + +class LoaderGenerator: + """Base loader generator for loader generators.""" + @abstractmethod + def generate_loaders(self, loader_pool): + """ + Abstract method for generating loaders. + + Args: + loader_pool (dict[str, LoaderStruct]): Current loader pool in data_manager. + + Returns: + dict[str, LoaderStruct], a dict of `Loader`. + + """ + + @abstractmethod + def check_train_job_exist(self, train_id): + """ + Abstract method for checking if train job exists. + + Args: + train_id (str): Train ID. + + Returns: + bool, if train job exists, return True. + + """ + + @abstractmethod + def generate_loader_by_train_id(self, train_id): + """ + Abstract method for generating loader by train id. + + Args: + train_id (str): Train ID. + + Returns: + dict[str, LoaderStruct], a dict of `Loader`. + + """ diff --git a/mindinsight/datavisual/data_transform/loader_generators/loader_struct.py b/mindinsight/datavisual/data_transform/loader_generators/loader_struct.py new file mode 100644 index 00000000..2f45587b --- /dev/null +++ b/mindinsight/datavisual/data_transform/loader_generators/loader_struct.py @@ -0,0 +1,64 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Loader struct.""" + + +class LoaderStruct: + """ + Loader to save summary info. + + LoaderStruct contains: loader_id, name, path, latest_update_time, status, data_loader. + """ + def __init__(self, loader_id, name, path, latest_update_time, data_loader): + self._loader_id = loader_id + self._name = name + self._path = path + self._latest_update_time = latest_update_time + self._data_loader = data_loader + + @property + def loader_id(self): + """Get loader ID.""" + return self._loader_id + + @property + def name(self): + """Get loader name.""" + return self._name + + @property + def latest_update_time(self): + """Get the latest update time of loader.""" + return self._latest_update_time + + @property + def data_loader(self): + """Get data loader.""" + return self._data_loader + + @latest_update_time.setter + def latest_update_time(self, latest_update_time): + """Set the latest update time of loader.""" + self._latest_update_time = latest_update_time + + def to_dict(self): + """Transform LoaderStruct to dict.""" + return dict( + loader_id=self._loader_id, + name=self._name, + path=self._path, + latest_update_time=self._latest_update_time, + data_loader=self._data_loader + ) diff --git a/mindinsight/datavisual/data_transform/ms_data_loader.py b/mindinsight/datavisual/data_transform/ms_data_loader.py new file mode 100644 index 00000000..8d9afbdf --- /dev/null +++ b/mindinsight/datavisual/data_transform/ms_data_loader.py @@ -0,0 +1,373 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +DataLoader for MindSpore data. + +This module is used to load the MindSpore training log file. +Each instance will read an entire run, a run can contain one or +more log file. +""" +import re +import struct + +from google.protobuf.message import DecodeError +from google.protobuf.text_format import ParseError + +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.data_access.file_handler import FileHandler +from mindinsight.datavisual.data_transform.events_data import EventsData +from mindinsight.datavisual.data_transform.events_data import TensorEvent +from mindinsight.datavisual.data_transform.graph import MSGraph +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 +from mindinsight.datavisual.proto_files import mindinsight_anf_ir_pb2 as anf_ir_pb2 +from mindinsight.datavisual.utils import crc32 +from mindinsight.utils.exceptions import UnknownError + +HEADER_SIZE = 8 +CRC_STR_SIZE = 4 + + +class MSDataLoader: + """ + MSDataLoader class, load MindSpore event data. + + Args: + summary_dir (str): Log directory. + """ + + def __init__(self, summary_dir): + self._init_instance(summary_dir) + + def _init_instance(self, summary_dir): + self._summary_dir = summary_dir + self._valid_filenames = [] + self._events_data = EventsData() + self._latest_summary_filename = '' + self._latest_summary_file_size = 0 + self._summary_file_handler = None + self._latest_pb_file_mtime = 0 + + def get_events_data(self): + """Return events data read from log file.""" + return self._events_data + + def _check_files_deleted(self, filenames, old_filenames): + """ + Check the file list for updates. + + Args: + filenames (list[str]): The latest files list. + old_filenames (list[str]): List of old files. + """ + deleted_files = set(old_filenames) - set(filenames) + if deleted_files: + logger.warning("There are some files has been deleted, " + "we will reload all files in path %s.", self._summary_dir) + self._init_instance(self._summary_dir) + + def load(self): + """ + Load all log valid files. + + When the file is reloaded, it will continue to load from where it left off. + """ + logger.debug("Start to load data in ms data loader.") + filenames = self.filter_valid_files() + if not filenames: + logger.warning("No valid files can be loaded, summary_dir: %s.", self._summary_dir) + raise exceptions.SummaryLogPathInvalid() + old_filenames = list(self._valid_filenames) + self._valid_filenames = filenames + self._check_files_deleted(filenames, old_filenames) + + self._load_summary_files(self._valid_filenames) + self._load_pb_files(self._valid_filenames) + + def _load_summary_files(self, filenames): + """ + Load summary file and parse file content. + + Args: + filenames (list[str]): File name list. + """ + summary_files = self._filter_summary_files(filenames) + summary_files = self._sorted_summary_files(summary_files) + + for filename in summary_files: + if self._latest_summary_filename and \ + (self._compare_summary_file(self._latest_summary_filename, filename)): + continue + + file_path = FileHandler.join(self._summary_dir, filename) + + if filename != self._latest_summary_filename: + self._summary_file_handler = FileHandler(file_path, 'rb') + self._latest_summary_filename = filename + self._latest_summary_file_size = 0 + + new_size = FileHandler.file_stat(file_path).size + if new_size == self._latest_summary_file_size: + continue + + self._latest_summary_file_size = new_size + try: + self._load_single_file(self._summary_file_handler) + except UnknownError as ex: + logger.warning("Parse summary file failed, detail: %r," + "file path: %s.", str(ex), file_path) + + def _load_single_file(self, file_handler): + """ + Load a log file data. + + Args: + file_handler (FileHandler): A file handler. + """ + logger.debug("Load single summary file, file path: %s.", file_handler.file_path) + while True: + start_offset = file_handler.offset + try: + event_str = self._event_load(file_handler) + if event_str is None: + file_handler.reset_offset(start_offset) + break + + event = summary_pb2.Event.FromString(event_str) + self._event_parse(event) + except exceptions.CRCFailedError: + file_handler.reset_offset(start_offset) + logger.warning("Check crc faild and ignore this file, file_path=%s, " + "offset=%s.", file_handler.file_path, file_handler.offset) + break + except (OSError, DecodeError, exceptions.MindInsightException) as ex: + logger.warning("Parse log file fail, and ignore this file, detail: %r," + "file path: %s.", str(ex), file_handler.file_path) + break + except Exception as ex: + logger.exception(ex) + raise UnknownError(str(ex)) + + def _event_load(self, file_handler): + """ + Load binary string to event string. + + Args: + file_handler (FileHandler): A file handler. + + Returns: + bytes, MindSpore event in bytes. + """ + # read the header + header_str = file_handler.read(HEADER_SIZE) + if not header_str: + logger.info("End of file, file_path=%s.", file_handler.file_path) + return None + header_crc_str = file_handler.read(CRC_STR_SIZE) + if not header_crc_str: + header_crc_str = '' + + if len(header_str) != HEADER_SIZE or len(header_crc_str) != CRC_STR_SIZE: + logger.warning("Check header size and crc, record truncated at offset %s, " + "file_path=%s.", file_handler.offset, file_handler.file_path) + return None + if crc32.GetValueFromStr(header_crc_str) != crc32.GetMaskCrc32cValue(header_str, HEADER_SIZE): + raise exceptions.CRCFailedError() + + # read the event body if integrity of header is verified + header = struct.unpack('Q', header_str) + event_len = int(header[0]) + + event_str = file_handler.read(event_len) + if not event_str: + event_str = '' + event_crc_str = file_handler.read(CRC_STR_SIZE) + if not event_crc_str: + event_crc_str = '' + + if len(event_str) != event_len or len(event_crc_str) != CRC_STR_SIZE: + logger.warning("Check event crc, record truncated at offset %d, file_path: %s.", + file_handler.offset, file_handler.file_path) + return None + if crc32.GetValueFromStr(event_crc_str) != crc32.GetMaskCrc32cValue(event_str, event_len): + raise exceptions.CRCFailedError() + + return event_str + + def _event_parse(self, event): + """ + Transform `Event` data to tensor_event and update it to EventsData. + + Args: + event (Event): Message event in summary proto, data read from file handler. + """ + if event.HasField('summary'): + for value in event.summary.value: + if value.HasField('scalar_value'): + tag = '{}/{}'.format(value.tag, PluginNameEnum.SCALAR.value) + tensor_event = TensorEvent(wall_time=event.wall_time, + step=event.step, + tag=tag, + plugin_name=PluginNameEnum.SCALAR.value, + value=value.scalar_value) + self._events_data.add_tensor_event(tensor_event) + + if value.HasField('image'): + tag = '{}/{}'.format(value.tag, PluginNameEnum.IMAGE.value) + tensor_event = TensorEvent(wall_time=event.wall_time, + step=event.step, + tag=tag, + plugin_name=PluginNameEnum.IMAGE.value, + value=value.image) + self._events_data.add_tensor_event(tensor_event) + + if event.HasField('graph_def'): + graph_proto = event.graph_def + graph = MSGraph() + graph.build_graph(graph_proto) + tensor_event = TensorEvent(wall_time=event.wall_time, + step=event.step, + tag=self._latest_summary_filename, + plugin_name=PluginNameEnum.GRAPH.value, + value=graph) + + try: + graph_tags = self._events_data.list_tags_by_plugin(PluginNameEnum.GRAPH.value) + except KeyError: + graph_tags = [] + summary_tags = self._filter_summary_files(graph_tags) + for tag in summary_tags: + self._events_data.delete_tensor_event(tag) + + self._events_data.add_tensor_event(tensor_event) + + def filter_valid_files(self): + """ + Gets a list of valid files from the given file path. + + Returns: + list[str], file name list. + + """ + filenames = [] + for filename in FileHandler.list_dir(self._summary_dir): + if FileHandler.is_file(FileHandler.join(self._summary_dir, filename)): + filenames.append(filename) + + valid_filenames = [] + valid_filenames.extend(self._filter_summary_files(filenames)) + valid_filenames.extend(self._filter_pb_files(filenames)) + return list(set(valid_filenames)) + + @staticmethod + def _filter_summary_files(filenames): + """ + Gets a list of summary files. + + Args: + filenames (list[str]): File name list, like [filename1, filename2]. + + Returns: + list[str], filename list. + """ + return list(filter( + lambda filename: (re.search(r'summary\.\d+', filename) + and not filename.endswith("_lineage")), filenames)) + + @staticmethod + def _compare_summary_file(current_file, dst_file): + """ + Compare the creation times of the two summary log files. + + Args: + current_file (str): Must be the summary log file path. + dst_file (str): Must be the summary log file path. + + Returns: + bool, returns True if the current file is new, or False if not. + """ + current_time = int(re.search(r'summary\.(\d+)', current_file)[1]) + dst_time = int(re.search(r'summary\.(\d+)', dst_file)[1]) + if current_time > dst_time or (current_time == dst_time and current_file > dst_file): + return True + return False + + @staticmethod + def _sorted_summary_files(summary_files): + """Sort by creating time increments and filenames decrement.""" + filenames = sorted(summary_files, + key=lambda filename: (-int(re.search(r'summary\.(\d+)', filename)[1]), filename), + reverse=True) + return filenames + + @staticmethod + def _filter_pb_files(filenames): + """ + Get a list of pb files. + + Args: + filenames (list[str]): File name list, like [filename1, filename2]. + + Returns: + list[str], filename list. + """ + return list(filter(lambda filename: re.search(r'\.pb$', filename), filenames)) + + def _load_pb_files(self, filenames): + """ + Load and parse the pb files. + + Args: + filenames (list[str]): File name list, like [filename1, filename2]. + + Returns: + list[str], filename list. + """ + pb_filenames = self._filter_pb_files(filenames) + pb_filenames = sorted(pb_filenames, key=lambda file: FileHandler.file_stat( + FileHandler.join(self._summary_dir, file)).mtime) + for filename in pb_filenames: + mtime = FileHandler.file_stat(FileHandler.join(self._summary_dir, filename)).mtime + if mtime <= self._latest_pb_file_mtime: + continue + self._latest_pb_file_mtime = mtime + self._parse_pb_file(filename) + + def _parse_pb_file(self, filename): + """ + Parse pb file and write content to `EventsData`. + + Args: + filename (str): The file path of pb file. + """ + file_path = FileHandler.join(self._summary_dir, filename) + logger.info("Start to load graph from pb file, file path: %s.", file_path) + filehandler = FileHandler(file_path) + model_proto = anf_ir_pb2.ModelProto() + try: + model_proto.ParseFromString(filehandler.read()) + except ParseError: + logger.warning("The given file is not a valid pb file, file path: %s.", file_path) + return + + graph = MSGraph() + graph.build_graph(model_proto.graph) + tensor_event = TensorEvent(wall_time=FileHandler.file_stat(file_path), + step=0, + tag=filename, + plugin_name=PluginNameEnum.GRAPH.value, + value=graph) + self._events_data.add_tensor_event(tensor_event) diff --git a/mindinsight/datavisual/data_transform/reservoir.py b/mindinsight/datavisual/data_transform/reservoir.py new file mode 100644 index 00000000..2e09ba47 --- /dev/null +++ b/mindinsight/datavisual/data_transform/reservoir.py @@ -0,0 +1,108 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""A reservoir sampling on the values.""" + +import random +import threading + +from mindinsight.utils.exceptions import ParamValueError + + +class Reservoir: + """ + A container based on Reservoir Sampling algorithm. + + The newly added sample will be preserved. If the container is full, an old + sample will be replaced randomly. The probability of each sample being + replaced is the same. + """ + + def __init__(self, size): + """ + A Container constructor which create a new Reservoir. + + Args: + size (int): Container Size. If the size is 0, the container is not limited. + + Raises: + ValueError: If size is negative integer. + """ + if not isinstance(size, (int,)) or size < 0: + raise ParamValueError('size must be nonnegative integer, was %s' % size) + + self._samples_max_size = size + self._samples = [] + self._sample_counter = 0 + self._sample_selector = random.Random(0) + self._mutex = threading.Lock() + + def samples(self): + """Return all stored samples.""" + with self._mutex: + return list(self._samples) + + def add_sample(self, sample): + """ + Add a sample to Reservoir. + + Replace the old sample when the capacity is full. + New added samples are guaranteed to be added to the reservoir. + + Args: + sample (Any): The sample to add to the Reservoir. + """ + with self._mutex: + if len(self._samples) < self._samples_max_size or self._samples_max_size == 0: + self._samples.append(sample) + else: + # Use the Reservoir Sampling algorithm to replace the old sample. + rand_int = self._sample_selector.randint( + 0, self._sample_counter) + if rand_int < self._samples_max_size: + self._samples.pop(rand_int) + self._samples.append(sample) + else: + self._samples[-1] = sample + self._sample_counter += 1 + + def remove_sample(self, filter_fun): + """ + Remove the samples from Reservoir that do not meet the filter criteria. + + Args: + filter_fun (Callable[..., Any]): Determines whether a sample meets + the deletion condition. + + Returns: + int, the number of samples removed. + """ + remove_size = 0 + + with self._mutex: + before_remove_size = len(self._samples) + if before_remove_size > 0: + # remove samples that meet the filter criteria. + self._samples = list(filter(filter_fun, self._samples)) + after_remove_size = len(self._samples) + remove_size = before_remove_size - after_remove_size + + if remove_size > 0: + # update _sample_counter when samples has been removed. + sample_remaining_rate = float( + after_remove_size) / before_remove_size + self._sample_counter = int( + round(self._sample_counter * sample_remaining_rate)) + + return remove_size diff --git a/mindinsight/datavisual/data_transform/summary_watcher.py b/mindinsight/datavisual/data_transform/summary_watcher.py new file mode 100644 index 00000000..8ba19acf --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_watcher.py @@ -0,0 +1,344 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Summary watcher module.""" + +import os +import re +import datetime +from pathlib import Path + +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.common.validation import Validation +from mindinsight.utils.exceptions import FileSystemPermissionError + + +class SummaryWatcher: + """SummaryWatcher class.""" + + SUMMARY_FILENAME_REGEX = r'summary\.(?P\d+)' + PB_FILENAME_REGEX = r'\.pb$' + MAX_SUMMARY_DIR_COUNT = 999 + + # scan at most 20000 files/directories (approximately 1 seconds) + # if overall=False in SummaryWatcher.list_summary_directories + # to avoid long-time blocking + MAX_SCAN_COUNT = 20000 + + def list_summary_directories(self, summary_base_dir, overall=True): + """ + List summary directories within base directory. + + Args: + summary_base_dir (str): Path of summary base directory. + + Returns: + list, list of summary directory info, each of which including the following attributes. + - relative_path (str): Relative path of summary directory, referring to settings.SUMMARY_BASE_DIR, + starting with "./". + - create_time (datetime): Creation time of summary file. + - update_time (datetime): Modification time of summary file. + + Examples: + >>> from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + >>> summary_watcher = SummaryWatcher() + >>> directories = summary_watcher.list_summary_directories('/summary/base/dir') + """ + if self._contains_null_byte(summary_base_dir=summary_base_dir): + return [] + + if not os.path.exists(summary_base_dir): + logger.warning('Path of summary base directory not exists.') + return [] + + if not os.path.isdir(summary_base_dir): + logger.warning('Path of summary base directory is not a valid directory.') + return [] + + summary_dict = {} + scan_count = 0 + + try: + entries = os.scandir(summary_base_dir) + except PermissionError: + logger.error('Path of summary base directory is not accessible.') + raise FileSystemPermissionError('Path of summary base directory is not accessible.') + + for entry in entries: + if len(summary_dict) == self.MAX_SUMMARY_DIR_COUNT: + break + relative_path = os.path.join('.', '') + if entry.is_symlink(): + pass + elif entry.is_file(): + self._update_summary_dict(summary_dict, relative_path, entry) + elif entry.is_dir(): + full_path = os.path.realpath(os.path.join(summary_base_dir, entry.name)) + + try: + subdir_entries = os.scandir(full_path) + except PermissionError: + logger.warning('Path of %s under summary base directory is not accessible.', entry.name) + else: + for subdir_entry in subdir_entries: + if len(summary_dict) == self.MAX_SUMMARY_DIR_COUNT: + break + subdir_relative_path = os.path.join('.', entry.name) + if subdir_entry.is_symlink(): + pass + elif subdir_entry.is_file(): + self._update_summary_dict(summary_dict, subdir_relative_path, subdir_entry) + + scan_count += 1 + if not overall and scan_count >= self.MAX_SCAN_COUNT: + break + + scan_count += 1 + if not overall and scan_count >= self.MAX_SCAN_COUNT: + logger.info('Stop further scanning due to overall is False and ' + 'number of scanned files exceeds upper limit.') + break + + directories = [{ + 'relative_path': key, + 'create_time': value['ctime'], + 'update_time': value['mtime'], + } for key, value in summary_dict.items()] + + # sort by update time in descending order and relative path in ascending order + directories.sort(key=lambda x: (-int(x['update_time'].timestamp()), x['relative_path'])) + + return directories + + def _contains_null_byte(self, **kwargs): + """ + Check if arg contains null byte. + + Args: + kwargs (Any): Check if arg contains null byte. + + Returns: + bool, indicates if any arg contains null byte. + """ + for key, value in kwargs.items(): + if not isinstance(value, str): + continue + if '\x00' in value: + logger.warning('%s contains null byte \\x00.', key) + return True + + return False + + def _is_valid_summary_directory(self, summary_base_dir, relative_path): + """ + Check if the given summary directory is valid. + + Args: + summary_base_dir (str): Path of summary base directory. + relative_path (str): Relative path of summary directory, referring to summary base directory, + starting with "./" . + + Returns: + bool, indicates if summary directory is valid. + """ + summary_base_dir = os.path.realpath(summary_base_dir) + summary_directory = os.path.realpath(os.path.join(summary_base_dir, relative_path)) + if summary_base_dir == summary_directory: + return True + + if not os.path.exists(summary_directory): + logger.warning('Path of summary directory not exists.') + return False + + if not os.path.isdir(summary_directory): + logger.warning('Path of summary directory is not a valid directory.') + return False + + try: + Path(summary_directory).relative_to(Path(summary_base_dir)) + except ValueError: + logger.warning('Relative path %s is not subdirectory of summary_base_dir', relative_path) + return False + + return True + + def _update_summary_dict(self, summary_dict, relative_path, entry): + """ + Update summary_dict with ctime and mtime. + + Args: + summary_dict (dict): Temporary data structure to hold summary directory info. + relative_path (str): Relative path of summary directory, referring to summary base directory, + starting with "./" . + entry (DirEntry): Directory entry instance needed to check with regular expression. + """ + summary_pattern = re.search(self.SUMMARY_FILENAME_REGEX, entry.name) + pb_pattern = re.search(self.PB_FILENAME_REGEX, entry.name) + if summary_pattern is None and pb_pattern is None: + return + + if summary_pattern is not None: + timestamp = int(summary_pattern.groupdict().get('timestamp')) + try: + # extract created time from filename + ctime = datetime.datetime.fromtimestamp(timestamp).astimezone() + except OverflowError: + return + else: + ctime = datetime.datetime.fromtimestamp(entry.stat().st_ctime).astimezone() + + # extract modified time from filesystem + mtime = datetime.datetime.fromtimestamp(entry.stat().st_mtime).astimezone() + + if relative_path not in summary_dict or summary_dict[relative_path]['ctime'] < ctime: + summary_dict[relative_path] = { + 'ctime': ctime, + 'mtime': mtime, + } + + def is_summary_directory(self, summary_base_dir, relative_path): + """ + Check if the given summary directory is valid. + + Args: + summary_base_dir (str): Path of summary base directory. + relative_path (str): Relative path of summary directory, referring to summary base directory, + starting with "./" . + + Returns: + bool, indicates if the given summary directory is valid. + + Examples: + >>> from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + >>> summary_watcher = SummaryWatcher() + >>> summaries = summary_watcher.is_summary_directory('/summary/base/dir', './job-01') + """ + if self._contains_null_byte(summary_base_dir=summary_base_dir, relative_path=relative_path): + return False + + if not self._is_valid_summary_directory(summary_base_dir, relative_path): + return False + + summary_directory = os.path.realpath(os.path.join(summary_base_dir, relative_path)) + try: + entries = os.scandir(summary_directory) + except PermissionError: + logger.error('Path of summary base directory is not accessible.') + raise FileSystemPermissionError('Path of summary base directory is not accessible.') + + for entry in entries: + if entry.is_symlink() or not entry.is_file(): + continue + summary_pattern = re.search(self.SUMMARY_FILENAME_REGEX, entry.name) + pb_pattern = re.search(self.PB_FILENAME_REGEX, entry.name) + if summary_pattern or pb_pattern: + return True + + return False + + def list_summary_directories_by_pagination(self, summary_base_dir, offset=0, limit=10): + """ + List summary directories within base directory. + + Args: + summary_base_dir (str): Path of summary base directory. + offset (int): An offset for page. Ex, offset is 0, mean current page is 1. Default value is 0. + limit (int): The max data items for per page. Default value is 10. + + Returns: + tuple[total, directories], total indicates the overall number of summary directories and directories + indicate list of summary directory info including the following attributes. + - relative_path (str): Relative path of summary directory, referring to settings.SUMMARY_BASE_DIR, + starting with "./". + - create_time (datetime): Creation time of summary file. + - update_time (datetime): Modification time of summary file. + + Raises: + ParamValueError, if offset < 0 or limit is out of valid value range. + ParamTypeError, if offset or limit is not valid integer. + + Examples: + >>> from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + >>> summary_watcher = SummaryWatcher() + >>> total, directories = summary_watcher.list_summary_directories_by_pagination( + '/summary/base/dir', offset=0, limit=10) + """ + offset = Validation.check_offset(offset=offset) + limit = Validation.check_limit(limit, min_value=1, max_value=999) + + directories = self.list_summary_directories(summary_base_dir, overall=False) + return len(directories), directories[offset * limit:(offset + 1) * limit] + + def list_summaries(self, summary_base_dir, relative_path='./'): + """ + Get info of latest summary file within the given summary directory. + + Args: + summary_base_dir (str): Path of summary base directory. + relative_path (str): Relative path of summary directory, referring to summary base directory, + starting with "./" . + + Returns: + list, list of summary file including the following attributes. + - file_name (str): Summary file name. + - create_time (datetime): Creation time of summary file. + - update_time (datetime): Modification time of summary file. + + Examples: + >>> from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + >>> summary_watcher = SummaryWatcher() + >>> summaries = summary_watcher.list_summaries('/summary/base/dir', './job-01') + """ + if self._contains_null_byte(summary_base_dir=summary_base_dir, relative_path=relative_path): + return [] + + if not self._is_valid_summary_directory(summary_base_dir, relative_path): + return [] + + summaries = [] + summary_directory = os.path.realpath(os.path.join(summary_base_dir, relative_path)) + try: + entries = os.scandir(summary_directory) + except PermissionError: + logger.error('Path of summary directory is not accessible.') + raise FileSystemPermissionError('Path of summary directory is not accessible.') + + for entry in entries: + if entry.is_symlink() or not entry.is_file(): + continue + + pattern = re.search(self.SUMMARY_FILENAME_REGEX, entry.name) + if pattern is None: + continue + + timestamp = int(pattern.groupdict().get('timestamp')) + try: + # extract created time from filename + ctime = datetime.datetime.fromtimestamp(timestamp).astimezone() + except OverflowError: + continue + + # extract modified time from filesystem + mtime = datetime.datetime.fromtimestamp(entry.stat().st_mtime).astimezone() + + summaries.append({ + 'file_name': entry.name, + 'create_time': ctime, + 'update_time': mtime, + }) + + # sort by update time in descending order and filename in ascending order + summaries.sort(key=lambda x: (-int(x['update_time'].timestamp()), x['file_name'])) + + return summaries diff --git a/mindinsight/datavisual/processors/__init__.py b/mindinsight/datavisual/processors/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/processors/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/processors/base_processor.py b/mindinsight/datavisual/processors/base_processor.py new file mode 100644 index 00000000..1849e61b --- /dev/null +++ b/mindinsight/datavisual/processors/base_processor.py @@ -0,0 +1,28 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base processor, and init data manager parameter.""" + + +class BaseProcessor: + """Base processors processor. All processors should inherit this class.""" + + def __init__(self, data_manager): + """ + Init image processor. + + Args: + data_manager (DataManager): A DataManager instance. + """ + self._data_manager = data_manager diff --git a/mindinsight/datavisual/processors/graph_processor.py b/mindinsight/datavisual/processors/graph_processor.py new file mode 100644 index 00000000..8ff73703 --- /dev/null +++ b/mindinsight/datavisual/processors/graph_processor.py @@ -0,0 +1,145 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +This file is to process `data_transform.data_manager` to handle graph, +and the status of graph will be checked before calling `Graph` object. +""" + +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.common.validation import Validation +from mindinsight.datavisual.data_transform.graph import NodeTypeEnum +from mindinsight.datavisual.processors.base_processor import BaseProcessor +from mindinsight.utils.exceptions import ParamValueError + + +class GraphProcessor(BaseProcessor): + """ + This object is to handle `DataManager` object, and process graph object. + + Args: + train_id (str): To get train job data by this given id. + data_manager (DataManager): A `DataManager` object. + tag (str): The tag of graph, if tag is None, will load the first graph. + """ + def __init__(self, train_id, data_manager, tag=None): + Validation.check_param_empty(train_id=train_id) + super(GraphProcessor, self).__init__(data_manager) + + train_job = self._data_manager.get_train_job_by_plugin(train_id, PluginNameEnum.GRAPH.value) + if train_job is None: + raise exceptions.SummaryLogPathInvalid() + if not train_job['tags']: + raise ParamValueError("Can not find any graph data in the train job.") + + if tag is None: + tag = train_job['tags'][0] + + tensors = self._data_manager.list_tensors(train_id, tag=tag) + self._graph = tensors[0].value + + def get_nodes(self, name, node_type): + """ + Get the nodes of every layer in graph. + + Args: + name (str): The name of a node. + node_type (Any): The type of node, either 'name_scope' or 'polymeric'. + + Returns: + TypedDict('Nodes', {'nodes': list[Node]}), format is {'nodes': []}. + example: + { + "nodes" : [ + { + "attr" : + { + "index" : "i: 0\n" + }, + "input" : {}, + "name" : "input_tensor", + "output" : + { + "Default/TensorAdd-op17" : + { + "edge_type" : "data", + "scope" : "name_scope", + "shape" : [1, 16, 128, 128] + } + }, + "output_i" : -1, + "polymeric_input" : {}, + "polymeric_output" : {}, + "polymeric_scope_name" : "", + "subnode_count" : 0, + "type" : "Data" + } + ] + } + """ + if node_type not in [NodeTypeEnum.NAME_SCOPE.value, NodeTypeEnum.POLYMERIC_SCOPE.value]: + raise ParamValueError( + 'The node type is not support, only either %s or %s.' + '' % (NodeTypeEnum.NAME_SCOPE.value, NodeTypeEnum.POLYMERIC_SCOPE.value)) + + if name and not self._graph.exist_node(name): + raise ParamValueError("The node name is not in graph.") + nodes = [] + if node_type == NodeTypeEnum.NAME_SCOPE.value: + nodes = self._graph.get_normal_nodes(name) + + if node_type == NodeTypeEnum.POLYMERIC_SCOPE.value: + if not name: + raise ParamValueError('The node name "%s" not in graph, node type is %s.' % + (name, node_type)) + polymeric_scope_name = name + nodes = self._graph.get_polymeric_nodes(polymeric_scope_name) + + return {'nodes': nodes} + + def search_node_names(self, search_content, offset, limit): + """ + Search node names by search content. + + Args: + search_content (Any): This content can be the key content of the node to search. + offset (int): An offset for page. Ex, offset is 0, mean current page is 1. + limit (int): The max data items for per page. + + Returns: + TypedDict('Names', {'names': list[str]}), {"names": ["node_names"]}. + """ + offset = Validation.check_offset(offset=offset) + limit = Validation.check_limit(limit, min_value=1, max_value=1000) + names = self._graph.search_node_names(search_content, offset, limit) + return {"names": names} + + def search_single_node(self, name): + """ + Search node by node name. + + Args: + name (str): The name of node. + + Returns: + dict, format is: + item_object = {'nodes': [], + 'scope_name': '', + 'children': {}} + """ + Validation.check_param_empty(name=name) + + nodes = self._graph.search_single_node(name) + return nodes diff --git a/mindinsight/datavisual/processors/images_processor.py b/mindinsight/datavisual/processors/images_processor.py new file mode 100644 index 00000000..fd52b54c --- /dev/null +++ b/mindinsight/datavisual/processors/images_processor.py @@ -0,0 +1,92 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Image Processor APIs.""" +from mindinsight.datavisual.utils.tools import to_int +from mindinsight.utils.exceptions import ParamValueError +from mindinsight.datavisual.common.validation import Validation +from mindinsight.datavisual.processors.base_processor import BaseProcessor + + +class ImageProcessor(BaseProcessor): + """Image Processor.""" + + def get_metadata_list(self, train_id, tag): + """ + Builds a JSON-serializable object with information about images. + + Args: + train_id (str): The ID of the events data. + tag (str): The name of the tag the images all belong to. + + Returns: + list[dict], a list of dictionaries containing the `wall_time`, `step`, `width`, + and `height` for each image. + [ + { + "wall_time": ****, + "step": ****, + "width": ****, + "height": ****, + }, + {...} + ] + + """ + Validation.check_param_empty(train_id=train_id, tag=tag) + result = [] + tensors = self._data_manager.list_tensors(train_id, tag) + + for tensor in tensors: + # no tensor_proto in TensorEvent + (width, height) = (tensor.value.width, tensor.value.height) + result.append({ + 'wall_time': tensor.wall_time, + 'step': tensor.step, + 'width': int(width), + 'height': int(height), + }) + return dict(metadatas=result) + + def get_single_image(self, train_id, tag, step): + """ + Returns the actual image bytes for a given image. + + Args: + train_id (str): The ID of the events data the image belongs to. + tag (str): The name of the tag the images belongs to. + step (int): The step of the image in the current reservoir. + + Returns: + bytes, a byte string of the raw image bytes. + + """ + Validation.check_param_empty(train_id=train_id, tag=tag, step=step) + step = to_int(step, "step") + + tensors = self._data_manager.list_tensors(train_id, tag) + + image = None + for tensor in tensors: + if tensor.step == step: + # Default value for bytes field is empty byte string normally, + # see also "Optional Fields And Default Values" in protobuf + # documentation. + image = tensor.value.encoded_image + break + + if image is None: + raise ParamValueError("Can not find the step with given train job id and tag.") + + return image diff --git a/mindinsight/datavisual/processors/scalars_processor.py b/mindinsight/datavisual/processors/scalars_processor.py new file mode 100644 index 00000000..e665c67c --- /dev/null +++ b/mindinsight/datavisual/processors/scalars_processor.py @@ -0,0 +1,43 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Scalar Processor APIs.""" +from mindinsight.datavisual.common.validation import Validation +from mindinsight.datavisual.processors.base_processor import BaseProcessor + + +class ScalarsProcessor(BaseProcessor): + """Scalar Processor.""" + + def get_metadata_list(self, train_id, tag): + """ + Builds a JSON-serializable object with information about scalars. + + Args: + train_id (str): The ID of the events data. + tag (str): The name of the tag the scalars all belonging to. + + Returns: + list[dict], a list of dictionaries containing the `wall_time`, `step`, `value` for each scalar. + """ + Validation.check_param_empty(train_id=train_id, tag=tag) + job_response = [] + tensors = self._data_manager.list_tensors(train_id, tag) + + for tensor in tensors: + job_response.append({ + 'wall_time': tensor.wall_time, + 'step': tensor.step, + 'value': tensor.value}) + return dict(metadatas=job_response) diff --git a/mindinsight/datavisual/processors/train_task_manager.py b/mindinsight/datavisual/processors/train_task_manager.py new file mode 100644 index 00000000..9d0ab253 --- /dev/null +++ b/mindinsight/datavisual/processors/train_task_manager.py @@ -0,0 +1,65 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train task manager.""" + +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.common.validation import Validation +from mindinsight.datavisual.processors.base_processor import BaseProcessor + + +class TrainTaskManager(BaseProcessor): + """Train task manager.""" + + def get_single_train_task(self, plugin_name, train_id): + """ + get single train task. + + Args: + plugin_name (str): Plugin name, refer `PluginNameEnum`. + train_id (str): Specify a training job to query. + + Returns: + {'train_jobs': list[TrainJob]}, refer to restful api. + """ + Validation.check_param_empty(plugin_name=plugin_name, train_id=train_id) + Validation.check_plugin_name(plugin_name=plugin_name) + train_job = self._data_manager.get_train_job_by_plugin(train_id=train_id, plugin_name=plugin_name) + if train_job is None: + raise exceptions.SummaryLogPathInvalid() + return dict(train_jobs=[train_job]) + + def get_plugins(self, train_id, manual_update=True): + """ + Queries the plug-in data for the specified training job + + Args: + train_id (str): Specify a training job to query. + manual_update (bool): Specifies whether to refresh automatically. + + Returns: + dict, refer to restful api. + """ + Validation.check_param_empty(train_id=train_id) + train_job = self._data_manager.get_single_train_job(train_id, manual_update=manual_update) + if not train_job: + default_result = dict() + for plugin_name in PluginNameEnum.list_members(): + default_result.update({plugin_name: list()}) + return dict(plugins=default_result) + + return dict( + plugins=train_job['tag_mapping'] + ) diff --git a/mindinsight/datavisual/proto_files/__init__.py b/mindinsight/datavisual/proto_files/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/datavisual/proto_files/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/datavisual/proto_files/mindinsight_anf_ir.proto b/mindinsight/datavisual/proto_files/mindinsight_anf_ir.proto new file mode 100644 index 00000000..721449ac --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_anf_ir.proto @@ -0,0 +1,328 @@ +// Copyright 2019 Huawei Technologies Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mindinsight; + + +// Versioning +enum Version { + // unknown version + UNKNOWWN_VERSION = 0; + + // Initial version (IR VERSION 1), published on Sep 23, 2019 + IR_VERSION = 0x0000000000000001; +} + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANYTHING = 40; // type anything +} + +// Value definiton for attribute value or parameter default value +message ValueProto { + // data type of value + optional DataType dtype = 1; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional bool bool_val = 2; // bool + optional int64 int_val = 3; // int + optional uint64 uint_val = 4; // uint + optional float float_val = 5; // float + optional double double_val = 6; // double + optional string str_val = 7; // string + optional TensorProto tensor_val = 8; // tensor value + optional GraphProto graph = 9; // graph + + repeated bool bool_vals = 10; // list of bool + repeated int64 int_vals = 11; // list of int + repeated uint64 uint_vals = 12; // list of uint + repeated float float_vals = 13; // list of float + repeated double double_vals = 14; // list of double + repeated string str_vals = 15; // list of string + repeated TensorProto tensor_vals = 16; // list of tensor value + repeated GraphProto graphs = 17; // list of graph + + // tuple or list + repeated ValueProto values = 18; // tuple, list of value + + // dictionary + repeated NamedValueProto dict_val = 19; // dictionary info + + // filed for type type + optional TypeProto type_val = 20; // type type info +} + +message AttributeProto { + optional string name = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +message NamedValueProto { + optional string key = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +// Defines a tensor shape. +message TensorShapeProto { + // One dimension of the tensor. + message Dimension { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + }; + + repeated Dimension dim = 1; +} + +// Types for graph input(parameter) and output +message TypeProto { + + message Tensor { + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType elem_type = 1; + optional TensorShapeProto shape = 2; // for scalar, this field is not set + } + + // tuple type + message Sequence { + // The type and optional shape of elements of the tuple. + repeated TypeProto elem_types = 1; + }; + + // data type + optional DataType data_type = 1; + + oneof value { + // The type of a tensor. + Tensor tensor_type = 2; + + // The type of a tuple. + Sequence sequence_type = 3; + } +} + +// Defines information on graph parameters, including the name, the type, and +// the default value of parameter if exists. +message ParameterProto { + optional string name = 1; // parameter name + optional TypeProto type = 2; // parameter type + optional ValueProto default_val = 3; // default value of parameter if exists +} + +// Defines graph output information +message OutputProto { + optional string name = 1; // output node name + optional TypeProto type = 2; // output node type +} + +// Define node input information +message InputProto { + enum EdgeType { + DATA_EDGE = 0; // data edge + CONTROL_EDGE = 1; // control edge + } + + optional string name = 1; + optional EdgeType type = 2; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated InputProto input = 1; // namespace Value + optional string name = 2; // namespace Value + + // The symbolic identifier of the Operator to execute. + optional string op_type = 3; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string scope = 4; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // Optional type info of this node + optional TypeProto output_type = 6; + + // other fields for debug + optional uint64 output_i = 7; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // ir version + optional int64 ir_version = 1; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 2; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 3; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 4; + + // metadata info of opeartors + optional OperatorSetProto metadata_operators = 5; +}; + +message OperatorProto { + optional string name = 1; // used as key, must be distinct + optional bytes config = 2; // operator config info + optional bytes obj_info = 3; // operator related object info, e.g. content of operator binary or name +}; + +message OperatorSetProto { + repeated OperatorProto operators = 1; + optional string summary = 2; // summary info of operators, e.g. file position of operators file +} + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // The parameters(inputs) and outputs of the graph. + repeated ParameterProto parameters = 3; + repeated OutputProto outputs = 4; + + // Constants used in this graph + repeated NamedValueProto const_vals = 5; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + // The shape of the tensor. + repeated int64 dims = 1; + + // The data type of the tensor. + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType data_type = 2; + + // Tensor content must be organized in row-major order. + // + // Depending on the data_type field, exactly one of the fields below with + // name ending in _data is used to store the elements of the tensor. + + // For float values + repeated float float_data = 3 [packed = true]; + + // For int32, uint8, int8, uint16, int16, and bool values + // When this field is present, the data_type field MUST be + // INT32, INT16, INT8, UINT16, UINT8, or BOOL + repeated int32 int32_data = 4 [packed = true]; + + // For int64. + // When this field is present, the data_type field MUST be INT64 + repeated int64 int64_data = 5 [packed = true]; + + // For double + // When this field is present, the data_type field MUST be DOUBLE + repeated double double_data = 6 [packed = true]; + + // For uint64 and uint32 values + // When this field is present, the data_type field MUST be + // UINT32 or UINT64 + repeated uint64 uint64_data = 7 [packed = true]; + + // Store raw tensor content. When this raw_data field is used to store tensor value, + // elements MUST be stored in as fixed-width, little-endian order. + optional bytes raw_data = 8; +} diff --git a/mindinsight/datavisual/proto_files/mindinsight_anf_ir_pb2.py b/mindinsight/datavisual/proto_files/mindinsight_anf_ir_pb2.py new file mode 100644 index 00000000..d049098c --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_anf_ir_pb2.py @@ -0,0 +1,1381 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: mindinsight_anf_ir.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='mindinsight_anf_ir.proto', + package='mindinsight', + syntax='proto2', + serialized_options=None, + serialized_pb=b'\n\x18mindinsight_anf_ir.proto\x12\x0bmindinsight\"\xc3\x04\n\nValueProto\x12$\n\x05\x64type\x18\x01 \x01(\x0e\x32\x15.mindinsight.DataType\x12\x10\n\x08\x62ool_val\x18\x02 \x01(\x08\x12\x0f\n\x07int_val\x18\x03 \x01(\x03\x12\x10\n\x08uint_val\x18\x04 \x01(\x04\x12\x11\n\tfloat_val\x18\x05 \x01(\x02\x12\x12\n\ndouble_val\x18\x06 \x01(\x01\x12\x0f\n\x07str_val\x18\x07 \x01(\t\x12,\n\ntensor_val\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProto\x12&\n\x05graph\x18\t \x01(\x0b\x32\x17.mindinsight.GraphProto\x12\x11\n\tbool_vals\x18\n \x03(\x08\x12\x10\n\x08int_vals\x18\x0b \x03(\x03\x12\x11\n\tuint_vals\x18\x0c \x03(\x04\x12\x12\n\nfloat_vals\x18\r \x03(\x02\x12\x13\n\x0b\x64ouble_vals\x18\x0e \x03(\x01\x12\x10\n\x08str_vals\x18\x0f \x03(\t\x12-\n\x0btensor_vals\x18\x10 \x03(\x0b\x32\x18.mindinsight.TensorProto\x12\'\n\x06graphs\x18\x11 \x03(\x0b\x32\x17.mindinsight.GraphProto\x12\'\n\x06values\x18\x12 \x03(\x0b\x32\x17.mindinsight.ValueProto\x12.\n\x08\x64ict_val\x18\x13 \x03(\x0b\x32\x1c.mindinsight.NamedValueProto\x12(\n\x08type_val\x18\x14 \x01(\x0b\x32\x16.mindinsight.TypeProto\"F\n\x0e\x41ttributeProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.mindinsight.ValueProto\"F\n\x0fNamedValueProto\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.mindinsight.ValueProto\"q\n\x10TensorShapeProto\x12\x34\n\x03\x64im\x18\x01 \x03(\x0b\x32\'.mindinsight.TensorShapeProto.Dimension\x1a\'\n\tDimension\x12\x0c\n\x04size\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xc8\x02\n\tTypeProto\x12(\n\tdata_type\x18\x01 \x01(\x0e\x32\x15.mindinsight.DataType\x12\x34\n\x0btensor_type\x18\x02 \x01(\x0b\x32\x1d.mindinsight.TypeProto.TensorH\x00\x12\x38\n\rsequence_type\x18\x03 \x01(\x0b\x32\x1f.mindinsight.TypeProto.SequenceH\x00\x1a`\n\x06Tensor\x12(\n\telem_type\x18\x01 \x01(\x0e\x32\x15.mindinsight.DataType\x12,\n\x05shape\x18\x02 \x01(\x0b\x32\x1d.mindinsight.TensorShapeProto\x1a\x36\n\x08Sequence\x12*\n\nelem_types\x18\x01 \x03(\x0b\x32\x16.mindinsight.TypeProtoB\x07\n\x05value\"r\n\x0eParameterProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x04type\x18\x02 \x01(\x0b\x32\x16.mindinsight.TypeProto\x12,\n\x0b\x64\x65\x66\x61ult_val\x18\x03 \x01(\x0b\x32\x17.mindinsight.ValueProto\"A\n\x0bOutputProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x04type\x18\x02 \x01(\x0b\x32\x16.mindinsight.TypeProto\"w\n\nInputProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x04type\x18\x02 \x01(\x0e\x32 .mindinsight.InputProto.EdgeType\"+\n\x08\x45\x64geType\x12\r\n\tDATA_EDGE\x10\x00\x12\x10\n\x0c\x43ONTROL_EDGE\x10\x01\"\xd0\x01\n\tNodeProto\x12&\n\x05input\x18\x01 \x03(\x0b\x32\x17.mindinsight.InputProto\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07op_type\x18\x03 \x01(\t\x12\r\n\x05scope\x18\x04 \x01(\t\x12.\n\tattribute\x18\x05 \x03(\x0b\x32\x1b.mindinsight.AttributeProto\x12+\n\x0boutput_type\x18\x06 \x01(\x0b\x32\x16.mindinsight.TypeProto\x12\x10\n\x08output_i\x18\x07 \x01(\x04\"\xaa\x01\n\nModelProto\x12\x12\n\nir_version\x18\x01 \x01(\x03\x12\x0e\n\x06\x64omain\x18\x02 \x01(\t\x12\x15\n\rmodel_version\x18\x03 \x01(\x03\x12&\n\x05graph\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProto\x12\x39\n\x12metadata_operators\x18\x05 \x01(\x0b\x32\x1d.mindinsight.OperatorSetProto\"?\n\rOperatorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x02 \x01(\x0c\x12\x10\n\x08obj_info\x18\x03 \x01(\x0c\"R\n\x10OperatorSetProto\x12-\n\toperators\x18\x01 \x03(\x0b\x32\x1a.mindinsight.OperatorProto\x12\x0f\n\x07summary\x18\x02 \x01(\t\"\xce\x01\n\nGraphProto\x12$\n\x04node\x18\x01 \x03(\x0b\x32\x16.mindinsight.NodeProto\x12\x0c\n\x04name\x18\x02 \x01(\t\x12/\n\nparameters\x18\x03 \x03(\x0b\x32\x1b.mindinsight.ParameterProto\x12)\n\x07outputs\x18\x04 \x03(\x0b\x32\x18.mindinsight.OutputProto\x12\x30\n\nconst_vals\x18\x05 \x03(\x0b\x32\x1c.mindinsight.NamedValueProto\"\xd1\x01\n\x0bTensorProto\x12\x0c\n\x04\x64ims\x18\x01 \x03(\x03\x12(\n\tdata_type\x18\x02 \x01(\x0e\x32\x15.mindinsight.DataType\x12\x16\n\nfloat_data\x18\x03 \x03(\x02\x42\x02\x10\x01\x12\x16\n\nint32_data\x18\x04 \x03(\x05\x42\x02\x10\x01\x12\x16\n\nint64_data\x18\x05 \x03(\x03\x42\x02\x10\x01\x12\x17\n\x0b\x64ouble_data\x18\x06 \x03(\x01\x42\x02\x10\x01\x12\x17\n\x0buint64_data\x18\x07 \x03(\x04\x42\x02\x10\x01\x12\x10\n\x08raw_data\x18\x08 \x01(\x0c*/\n\x07Version\x12\x14\n\x10UNKNOWWN_VERSION\x10\x00\x12\x0e\n\nIR_VERSION\x10\x01*\xfb\x04\n\x08\x44\x61taType\x12\x10\n\x0c\x44T_UNDEFINED\x10\x00\x12\x0b\n\x07\x44T_BOOL\x10\x01\x12\x0b\n\x07\x44T_INT8\x10\x02\x12\x0c\n\x08\x44T_INT16\x10\x03\x12\x0c\n\x08\x44T_INT32\x10\x04\x12\x0c\n\x08\x44T_INT64\x10\x05\x12\x0c\n\x08\x44T_UINT8\x10\x06\x12\r\n\tDT_UINT16\x10\x07\x12\r\n\tDT_UINT32\x10\x08\x12\r\n\tDT_UINT64\x10\t\x12\x0e\n\nDT_FLOAT16\x10\n\x12\x0e\n\nDT_FLOAT32\x10\x0b\x12\x0e\n\nDT_FLOAT64\x10\x0c\x12\r\n\tDT_STRING\x10\r\x12\r\n\tDT_TENSOR\x10\x0e\x12\x0c\n\x08\x44T_GRAPH\x10\x0f\x12\x0c\n\x08\x44T_BOOLS\x10\x10\x12\x0c\n\x08\x44T_INTS8\x10\x11\x12\r\n\tDT_INTS16\x10\x12\x12\r\n\tDT_INTS32\x10\x13\x12\r\n\tDT_INTS64\x10\x14\x12\r\n\tDT_UINTS8\x10\x15\x12\x0e\n\nDT_UINTS16\x10\x16\x12\x0e\n\nDT_UINTS32\x10\x17\x12\x0e\n\nDT_UINTS64\x10\x18\x12\x0f\n\x0b\x44T_FLOATS16\x10\x19\x12\x0f\n\x0b\x44T_FLOATS32\x10\x1a\x12\x0f\n\x0b\x44T_FLOATS64\x10\x1b\x12\x0e\n\nDT_STRINGS\x10\x1c\x12\x0e\n\nDT_TENSORS\x10\x1d\x12\r\n\tDT_GRAPHS\x10\x1e\x12\x0c\n\x08\x44T_TUPLE\x10\x1f\x12\x0b\n\x07\x44T_LIST\x10 \x12\x0b\n\x07\x44T_DICT\x10!\x12\x0b\n\x07\x44T_NONE\x10\"\x12\x0f\n\x0b\x44T_SYM_INST\x10#\x12\x0f\n\x0b\x44T_BASE_INT\x10$\x12\x10\n\x0c\x44T_BASE_UINT\x10%\x12\x11\n\rDT_BASE_FLOAT\x10&\x12\x0b\n\x07\x44T_TYPE\x10\'\x12\x0f\n\x0b\x44T_ANYTHING\x10(' +) + +_VERSION = _descriptor.EnumDescriptor( + name='Version', + full_name='mindinsight.Version', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='UNKNOWWN_VERSION', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='IR_VERSION', index=1, number=1, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=2471, + serialized_end=2518, +) +_sym_db.RegisterEnumDescriptor(_VERSION) + +Version = enum_type_wrapper.EnumTypeWrapper(_VERSION) +_DATATYPE = _descriptor.EnumDescriptor( + name='DataType', + full_name='mindinsight.DataType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='DT_UNDEFINED', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_BOOL', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INT8', index=2, number=2, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INT16', index=3, number=3, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INT32', index=4, number=4, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INT64', index=5, number=5, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINT8', index=6, number=6, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINT16', index=7, number=7, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINT32', index=8, number=8, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINT64', index=9, number=9, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOAT16', index=10, number=10, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOAT32', index=11, number=11, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOAT64', index=12, number=12, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_STRING', index=13, number=13, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_TENSOR', index=14, number=14, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_GRAPH', index=15, number=15, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_BOOLS', index=16, number=16, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INTS8', index=17, number=17, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INTS16', index=18, number=18, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INTS32', index=19, number=19, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_INTS64', index=20, number=20, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINTS8', index=21, number=21, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINTS16', index=22, number=22, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINTS32', index=23, number=23, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_UINTS64', index=24, number=24, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOATS16', index=25, number=25, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOATS32', index=26, number=26, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_FLOATS64', index=27, number=27, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_STRINGS', index=28, number=28, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_TENSORS', index=29, number=29, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_GRAPHS', index=30, number=30, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_TUPLE', index=31, number=31, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_LIST', index=32, number=32, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_DICT', index=33, number=33, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_NONE', index=34, number=34, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_SYM_INST', index=35, number=35, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_BASE_INT', index=36, number=36, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_BASE_UINT', index=37, number=37, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_BASE_FLOAT', index=38, number=38, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_TYPE', index=39, number=39, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DT_ANYTHING', index=40, number=40, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=2521, + serialized_end=3156, +) +_sym_db.RegisterEnumDescriptor(_DATATYPE) + +DataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE) +UNKNOWWN_VERSION = 0 +IR_VERSION = 1 +DT_UNDEFINED = 0 +DT_BOOL = 1 +DT_INT8 = 2 +DT_INT16 = 3 +DT_INT32 = 4 +DT_INT64 = 5 +DT_UINT8 = 6 +DT_UINT16 = 7 +DT_UINT32 = 8 +DT_UINT64 = 9 +DT_FLOAT16 = 10 +DT_FLOAT32 = 11 +DT_FLOAT64 = 12 +DT_STRING = 13 +DT_TENSOR = 14 +DT_GRAPH = 15 +DT_BOOLS = 16 +DT_INTS8 = 17 +DT_INTS16 = 18 +DT_INTS32 = 19 +DT_INTS64 = 20 +DT_UINTS8 = 21 +DT_UINTS16 = 22 +DT_UINTS32 = 23 +DT_UINTS64 = 24 +DT_FLOATS16 = 25 +DT_FLOATS32 = 26 +DT_FLOATS64 = 27 +DT_STRINGS = 28 +DT_TENSORS = 29 +DT_GRAPHS = 30 +DT_TUPLE = 31 +DT_LIST = 32 +DT_DICT = 33 +DT_NONE = 34 +DT_SYM_INST = 35 +DT_BASE_INT = 36 +DT_BASE_UINT = 37 +DT_BASE_FLOAT = 38 +DT_TYPE = 39 +DT_ANYTHING = 40 + + +_INPUTPROTO_EDGETYPE = _descriptor.EnumDescriptor( + name='EdgeType', + full_name='mindinsight.InputProto.EdgeType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='DATA_EDGE', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='CONTROL_EDGE', index=1, number=1, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=1472, + serialized_end=1515, +) +_sym_db.RegisterEnumDescriptor(_INPUTPROTO_EDGETYPE) + + +_VALUEPROTO = _descriptor.Descriptor( + name='ValueProto', + full_name='mindinsight.ValueProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dtype', full_name='mindinsight.ValueProto.dtype', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_val', full_name='mindinsight.ValueProto.bool_val', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_val', full_name='mindinsight.ValueProto.int_val', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='uint_val', full_name='mindinsight.ValueProto.uint_val', index=3, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_val', full_name='mindinsight.ValueProto.float_val', index=4, + number=5, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='double_val', full_name='mindinsight.ValueProto.double_val', index=5, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='str_val', full_name='mindinsight.ValueProto.str_val', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor_val', full_name='mindinsight.ValueProto.tensor_val', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graph', full_name='mindinsight.ValueProto.graph', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bool_vals', full_name='mindinsight.ValueProto.bool_vals', index=9, + number=10, type=8, cpp_type=7, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int_vals', full_name='mindinsight.ValueProto.int_vals', index=10, + number=11, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='uint_vals', full_name='mindinsight.ValueProto.uint_vals', index=11, + number=12, type=4, cpp_type=4, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_vals', full_name='mindinsight.ValueProto.float_vals', index=12, + number=13, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='double_vals', full_name='mindinsight.ValueProto.double_vals', index=13, + number=14, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='str_vals', full_name='mindinsight.ValueProto.str_vals', index=14, + number=15, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor_vals', full_name='mindinsight.ValueProto.tensor_vals', index=15, + number=16, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graphs', full_name='mindinsight.ValueProto.graphs', index=16, + number=17, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='values', full_name='mindinsight.ValueProto.values', index=17, + number=18, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dict_val', full_name='mindinsight.ValueProto.dict_val', index=18, + number=19, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type_val', full_name='mindinsight.ValueProto.type_val', index=19, + number=20, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=42, + serialized_end=621, +) + + +_ATTRIBUTEPROTO = _descriptor.Descriptor( + name='AttributeProto', + full_name='mindinsight.AttributeProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.AttributeProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.AttributeProto.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=623, + serialized_end=693, +) + + +_NAMEDVALUEPROTO = _descriptor.Descriptor( + name='NamedValueProto', + full_name='mindinsight.NamedValueProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.NamedValueProto.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.NamedValueProto.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=695, + serialized_end=765, +) + + +_TENSORSHAPEPROTO_DIMENSION = _descriptor.Descriptor( + name='Dimension', + full_name='mindinsight.TensorShapeProto.Dimension', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='size', full_name='mindinsight.TensorShapeProto.Dimension.size', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.TensorShapeProto.Dimension.name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=841, + serialized_end=880, +) + +_TENSORSHAPEPROTO = _descriptor.Descriptor( + name='TensorShapeProto', + full_name='mindinsight.TensorShapeProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dim', full_name='mindinsight.TensorShapeProto.dim', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TENSORSHAPEPROTO_DIMENSION, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=767, + serialized_end=880, +) + + +_TYPEPROTO_TENSOR = _descriptor.Descriptor( + name='Tensor', + full_name='mindinsight.TypeProto.Tensor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='elem_type', full_name='mindinsight.TypeProto.Tensor.elem_type', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shape', full_name='mindinsight.TypeProto.Tensor.shape', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1050, + serialized_end=1146, +) + +_TYPEPROTO_SEQUENCE = _descriptor.Descriptor( + name='Sequence', + full_name='mindinsight.TypeProto.Sequence', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='elem_types', full_name='mindinsight.TypeProto.Sequence.elem_types', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1148, + serialized_end=1202, +) + +_TYPEPROTO = _descriptor.Descriptor( + name='TypeProto', + full_name='mindinsight.TypeProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='data_type', full_name='mindinsight.TypeProto.data_type', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor_type', full_name='mindinsight.TypeProto.tensor_type', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sequence_type', full_name='mindinsight.TypeProto.sequence_type', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TYPEPROTO_TENSOR, _TYPEPROTO_SEQUENCE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='value', full_name='mindinsight.TypeProto.value', + index=0, containing_type=None, fields=[]), + ], + serialized_start=883, + serialized_end=1211, +) + + +_PARAMETERPROTO = _descriptor.Descriptor( + name='ParameterProto', + full_name='mindinsight.ParameterProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.ParameterProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type', full_name='mindinsight.ParameterProto.type', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='default_val', full_name='mindinsight.ParameterProto.default_val', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1213, + serialized_end=1327, +) + + +_OUTPUTPROTO = _descriptor.Descriptor( + name='OutputProto', + full_name='mindinsight.OutputProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.OutputProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type', full_name='mindinsight.OutputProto.type', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1329, + serialized_end=1394, +) + + +_INPUTPROTO = _descriptor.Descriptor( + name='InputProto', + full_name='mindinsight.InputProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.InputProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type', full_name='mindinsight.InputProto.type', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _INPUTPROTO_EDGETYPE, + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1396, + serialized_end=1515, +) + + +_NODEPROTO = _descriptor.Descriptor( + name='NodeProto', + full_name='mindinsight.NodeProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='input', full_name='mindinsight.NodeProto.input', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.NodeProto.name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='op_type', full_name='mindinsight.NodeProto.op_type', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='scope', full_name='mindinsight.NodeProto.scope', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='attribute', full_name='mindinsight.NodeProto.attribute', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_type', full_name='mindinsight.NodeProto.output_type', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_i', full_name='mindinsight.NodeProto.output_i', index=6, + number=7, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1518, + serialized_end=1726, +) + + +_MODELPROTO = _descriptor.Descriptor( + name='ModelProto', + full_name='mindinsight.ModelProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='ir_version', full_name='mindinsight.ModelProto.ir_version', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='domain', full_name='mindinsight.ModelProto.domain', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='model_version', full_name='mindinsight.ModelProto.model_version', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graph', full_name='mindinsight.ModelProto.graph', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='metadata_operators', full_name='mindinsight.ModelProto.metadata_operators', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1729, + serialized_end=1899, +) + + +_OPERATORPROTO = _descriptor.Descriptor( + name='OperatorProto', + full_name='mindinsight.OperatorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.OperatorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='config', full_name='mindinsight.OperatorProto.config', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='obj_info', full_name='mindinsight.OperatorProto.obj_info', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1901, + serialized_end=1964, +) + + +_OPERATORSETPROTO = _descriptor.Descriptor( + name='OperatorSetProto', + full_name='mindinsight.OperatorSetProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='operators', full_name='mindinsight.OperatorSetProto.operators', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='summary', full_name='mindinsight.OperatorSetProto.summary', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1966, + serialized_end=2048, +) + + +_GRAPHPROTO = _descriptor.Descriptor( + name='GraphProto', + full_name='mindinsight.GraphProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='node', full_name='mindinsight.GraphProto.node', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='name', full_name='mindinsight.GraphProto.name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='parameters', full_name='mindinsight.GraphProto.parameters', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='outputs', full_name='mindinsight.GraphProto.outputs', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='const_vals', full_name='mindinsight.GraphProto.const_vals', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2051, + serialized_end=2257, +) + + +_TENSORPROTO = _descriptor.Descriptor( + name='TensorProto', + full_name='mindinsight.TensorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dims', full_name='mindinsight.TensorProto.dims', index=0, + number=1, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data_type', full_name='mindinsight.TensorProto.data_type', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='float_data', full_name='mindinsight.TensorProto.float_data', index=2, + number=3, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int32_data', full_name='mindinsight.TensorProto.int32_data', index=3, + number=4, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='int64_data', full_name='mindinsight.TensorProto.int64_data', index=4, + number=5, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='double_data', full_name='mindinsight.TensorProto.double_data', index=5, + number=6, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='uint64_data', full_name='mindinsight.TensorProto.uint64_data', index=6, + number=7, type=4, cpp_type=4, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='raw_data', full_name='mindinsight.TensorProto.raw_data', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2260, + serialized_end=2469, +) + +_VALUEPROTO.fields_by_name['dtype'].enum_type = _DATATYPE +_VALUEPROTO.fields_by_name['tensor_val'].message_type = _TENSORPROTO +_VALUEPROTO.fields_by_name['graph'].message_type = _GRAPHPROTO +_VALUEPROTO.fields_by_name['tensor_vals'].message_type = _TENSORPROTO +_VALUEPROTO.fields_by_name['graphs'].message_type = _GRAPHPROTO +_VALUEPROTO.fields_by_name['values'].message_type = _VALUEPROTO +_VALUEPROTO.fields_by_name['dict_val'].message_type = _NAMEDVALUEPROTO +_VALUEPROTO.fields_by_name['type_val'].message_type = _TYPEPROTO +_ATTRIBUTEPROTO.fields_by_name['value'].message_type = _VALUEPROTO +_NAMEDVALUEPROTO.fields_by_name['value'].message_type = _VALUEPROTO +_TENSORSHAPEPROTO_DIMENSION.containing_type = _TENSORSHAPEPROTO +_TENSORSHAPEPROTO.fields_by_name['dim'].message_type = _TENSORSHAPEPROTO_DIMENSION +_TYPEPROTO_TENSOR.fields_by_name['elem_type'].enum_type = _DATATYPE +_TYPEPROTO_TENSOR.fields_by_name['shape'].message_type = _TENSORSHAPEPROTO +_TYPEPROTO_TENSOR.containing_type = _TYPEPROTO +_TYPEPROTO_SEQUENCE.fields_by_name['elem_types'].message_type = _TYPEPROTO +_TYPEPROTO_SEQUENCE.containing_type = _TYPEPROTO +_TYPEPROTO.fields_by_name['data_type'].enum_type = _DATATYPE +_TYPEPROTO.fields_by_name['tensor_type'].message_type = _TYPEPROTO_TENSOR +_TYPEPROTO.fields_by_name['sequence_type'].message_type = _TYPEPROTO_SEQUENCE +_TYPEPROTO.oneofs_by_name['value'].fields.append( + _TYPEPROTO.fields_by_name['tensor_type']) +_TYPEPROTO.fields_by_name['tensor_type'].containing_oneof = _TYPEPROTO.oneofs_by_name['value'] +_TYPEPROTO.oneofs_by_name['value'].fields.append( + _TYPEPROTO.fields_by_name['sequence_type']) +_TYPEPROTO.fields_by_name['sequence_type'].containing_oneof = _TYPEPROTO.oneofs_by_name['value'] +_PARAMETERPROTO.fields_by_name['type'].message_type = _TYPEPROTO +_PARAMETERPROTO.fields_by_name['default_val'].message_type = _VALUEPROTO +_OUTPUTPROTO.fields_by_name['type'].message_type = _TYPEPROTO +_INPUTPROTO.fields_by_name['type'].enum_type = _INPUTPROTO_EDGETYPE +_INPUTPROTO_EDGETYPE.containing_type = _INPUTPROTO +_NODEPROTO.fields_by_name['input'].message_type = _INPUTPROTO +_NODEPROTO.fields_by_name['attribute'].message_type = _ATTRIBUTEPROTO +_NODEPROTO.fields_by_name['output_type'].message_type = _TYPEPROTO +_MODELPROTO.fields_by_name['graph'].message_type = _GRAPHPROTO +_MODELPROTO.fields_by_name['metadata_operators'].message_type = _OPERATORSETPROTO +_OPERATORSETPROTO.fields_by_name['operators'].message_type = _OPERATORPROTO +_GRAPHPROTO.fields_by_name['node'].message_type = _NODEPROTO +_GRAPHPROTO.fields_by_name['parameters'].message_type = _PARAMETERPROTO +_GRAPHPROTO.fields_by_name['outputs'].message_type = _OUTPUTPROTO +_GRAPHPROTO.fields_by_name['const_vals'].message_type = _NAMEDVALUEPROTO +_TENSORPROTO.fields_by_name['data_type'].enum_type = _DATATYPE +DESCRIPTOR.message_types_by_name['ValueProto'] = _VALUEPROTO +DESCRIPTOR.message_types_by_name['AttributeProto'] = _ATTRIBUTEPROTO +DESCRIPTOR.message_types_by_name['NamedValueProto'] = _NAMEDVALUEPROTO +DESCRIPTOR.message_types_by_name['TensorShapeProto'] = _TENSORSHAPEPROTO +DESCRIPTOR.message_types_by_name['TypeProto'] = _TYPEPROTO +DESCRIPTOR.message_types_by_name['ParameterProto'] = _PARAMETERPROTO +DESCRIPTOR.message_types_by_name['OutputProto'] = _OUTPUTPROTO +DESCRIPTOR.message_types_by_name['InputProto'] = _INPUTPROTO +DESCRIPTOR.message_types_by_name['NodeProto'] = _NODEPROTO +DESCRIPTOR.message_types_by_name['ModelProto'] = _MODELPROTO +DESCRIPTOR.message_types_by_name['OperatorProto'] = _OPERATORPROTO +DESCRIPTOR.message_types_by_name['OperatorSetProto'] = _OPERATORSETPROTO +DESCRIPTOR.message_types_by_name['GraphProto'] = _GRAPHPROTO +DESCRIPTOR.message_types_by_name['TensorProto'] = _TENSORPROTO +DESCRIPTOR.enum_types_by_name['Version'] = _VERSION +DESCRIPTOR.enum_types_by_name['DataType'] = _DATATYPE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ValueProto = _reflection.GeneratedProtocolMessageType('ValueProto', (_message.Message,), { + 'DESCRIPTOR' : _VALUEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.ValueProto) + }) +_sym_db.RegisterMessage(ValueProto) + +AttributeProto = _reflection.GeneratedProtocolMessageType('AttributeProto', (_message.Message,), { + 'DESCRIPTOR' : _ATTRIBUTEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.AttributeProto) + }) +_sym_db.RegisterMessage(AttributeProto) + +NamedValueProto = _reflection.GeneratedProtocolMessageType('NamedValueProto', (_message.Message,), { + 'DESCRIPTOR' : _NAMEDVALUEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.NamedValueProto) + }) +_sym_db.RegisterMessage(NamedValueProto) + +TensorShapeProto = _reflection.GeneratedProtocolMessageType('TensorShapeProto', (_message.Message,), { + + 'Dimension' : _reflection.GeneratedProtocolMessageType('Dimension', (_message.Message,), { + 'DESCRIPTOR' : _TENSORSHAPEPROTO_DIMENSION, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TensorShapeProto.Dimension) + }) + , + 'DESCRIPTOR' : _TENSORSHAPEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TensorShapeProto) + }) +_sym_db.RegisterMessage(TensorShapeProto) +_sym_db.RegisterMessage(TensorShapeProto.Dimension) + +TypeProto = _reflection.GeneratedProtocolMessageType('TypeProto', (_message.Message,), { + + 'Tensor' : _reflection.GeneratedProtocolMessageType('Tensor', (_message.Message,), { + 'DESCRIPTOR' : _TYPEPROTO_TENSOR, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TypeProto.Tensor) + }) + , + + 'Sequence' : _reflection.GeneratedProtocolMessageType('Sequence', (_message.Message,), { + 'DESCRIPTOR' : _TYPEPROTO_SEQUENCE, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TypeProto.Sequence) + }) + , + 'DESCRIPTOR' : _TYPEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TypeProto) + }) +_sym_db.RegisterMessage(TypeProto) +_sym_db.RegisterMessage(TypeProto.Tensor) +_sym_db.RegisterMessage(TypeProto.Sequence) + +ParameterProto = _reflection.GeneratedProtocolMessageType('ParameterProto', (_message.Message,), { + 'DESCRIPTOR' : _PARAMETERPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.ParameterProto) + }) +_sym_db.RegisterMessage(ParameterProto) + +OutputProto = _reflection.GeneratedProtocolMessageType('OutputProto', (_message.Message,), { + 'DESCRIPTOR' : _OUTPUTPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OutputProto) + }) +_sym_db.RegisterMessage(OutputProto) + +InputProto = _reflection.GeneratedProtocolMessageType('InputProto', (_message.Message,), { + 'DESCRIPTOR' : _INPUTPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.InputProto) + }) +_sym_db.RegisterMessage(InputProto) + +NodeProto = _reflection.GeneratedProtocolMessageType('NodeProto', (_message.Message,), { + 'DESCRIPTOR' : _NODEPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.NodeProto) + }) +_sym_db.RegisterMessage(NodeProto) + +ModelProto = _reflection.GeneratedProtocolMessageType('ModelProto', (_message.Message,), { + 'DESCRIPTOR' : _MODELPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.ModelProto) + }) +_sym_db.RegisterMessage(ModelProto) + +OperatorProto = _reflection.GeneratedProtocolMessageType('OperatorProto', (_message.Message,), { + 'DESCRIPTOR' : _OPERATORPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperatorProto) + }) +_sym_db.RegisterMessage(OperatorProto) + +OperatorSetProto = _reflection.GeneratedProtocolMessageType('OperatorSetProto', (_message.Message,), { + 'DESCRIPTOR' : _OPERATORSETPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperatorSetProto) + }) +_sym_db.RegisterMessage(OperatorSetProto) + +GraphProto = _reflection.GeneratedProtocolMessageType('GraphProto', (_message.Message,), { + 'DESCRIPTOR' : _GRAPHPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.GraphProto) + }) +_sym_db.RegisterMessage(GraphProto) + +TensorProto = _reflection.GeneratedProtocolMessageType('TensorProto', (_message.Message,), { + 'DESCRIPTOR' : _TENSORPROTO, + '__module__' : 'mindinsight_anf_ir_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TensorProto) + }) +_sym_db.RegisterMessage(TensorProto) + + +_TENSORPROTO.fields_by_name['float_data']._options = None +_TENSORPROTO.fields_by_name['int32_data']._options = None +_TENSORPROTO.fields_by_name['int64_data']._options = None +_TENSORPROTO.fields_by_name['double_data']._options = None +_TENSORPROTO.fields_by_name['uint64_data']._options = None +# @@protoc_insertion_point(module_scope) diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary.proto b/mindinsight/datavisual/proto_files/mindinsight_summary.proto new file mode 100644 index 00000000..2b3b3d3a --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_summary.proto @@ -0,0 +1,155 @@ +// Copyright 2019 Huawei Technologies Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mindinsight; +option cc_enable_arenas = true; + +// The ANF IR define, include the tensor and graph define +import "mindinsight_anf_ir.proto"; + +// Event Protocol buffer, Top define +message Event { + // Timestamp + required double wall_time = 1; + + // The step of train. + optional int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // Now version is "Mindspore.Event:1" + string version = 3; + + // GraphDef. + GraphProto graph_def = 4; + + // Summary data + Summary summary = 5; + + // Train lineage + TrainLineage train_lineage = 6; + + // Evaluation lineage + EvaluationLineage evaluation_lineage = 7; + + // dataset graph + DatasetGraph dataset_graph = 9; + } +} + +// TrainLineage records infos of a train. +message TrainLineage{ + message HyperParameters{ + optional string optimizer = 1; + optional float learning_rate = 2; + optional string loss_function = 3; + optional int32 epoch = 4; + optional string parallel_mode = 5; + optional int32 device_num = 6; + optional int32 batch_size = 8; + } + + message TrainDataset{ + optional string train_dataset_path = 1; + optional int32 train_dataset_size = 2; + } + + message Algorithm{ + optional string network = 1; + optional float loss = 2; + } + + message Model{ + optional string path = 3; + optional int64 size = 4; + } + + optional HyperParameters hyper_parameters = 1; + optional TrainDataset train_dataset = 2; + optional Algorithm algorithm = 3; + optional Model model = 4; +} + +//EvalLineage records infos of evaluation. +message EvaluationLineage{ + message ValidDataset{ + optional string valid_dataset_path = 1; + optional int32 valid_dataset_size = 2; + } + + optional string metric = 2; + optional ValidDataset valid_dataset = 3; +} + +// A Summary is a set of named values that be produced regularly during training +message Summary { + message Image { + // Dimensions of the image. + required int32 height = 1; + required int32 width = 2; + // Valid colorspace values are + // 1 - grayscale + // 2 - grayscale + alpha + // 3 - RGB + // 4 - RGBA + // 5 - DIGITAL_YUV + // 6 - BGRA + required int32 colorspace = 3; + // Image data in encoded format. Now only support the RGB. + required bytes encoded_image = 4; + } + + message Value { + // Tag name for the data. + required string tag = 1; + + // Value associated with the tag. + oneof value { + float scalar_value = 3; + Image image = 4; + TensorProto tensor = 8; + } + } + + // Set of values for the summary. + repeated Value value = 1; +} + +// DatasetGraph +message DatasetGraph { + repeated DatasetGraph children = 1; + optional OperationParameter parameter = 2; + repeated Operation operations = 3; + optional Operation sampler = 4; +} + +message Operation { + optional OperationParameter operationParam = 1; + repeated int32 size = 2; + repeated float weights = 3; +} + +message OperationParameter{ + map mapStr = 1; + map mapStrList = 2; + map mapBool = 3; + map mapInt = 4; + map mapDouble = 5; +} + +message StrList { + repeated string strValue = 1; +} diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py new file mode 100644 index 00000000..f3b25906 --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py @@ -0,0 +1,1161 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: mindinsight_summary.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from mindinsight.datavisual.proto_files import mindinsight_anf_ir_pb2 as mindinsight__anf__ir__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='mindinsight_summary.proto', + package='mindinsight', + syntax='proto2', + serialized_options=b'\370\001\001', + serialized_pb=b'\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\xc0\x02\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x12\x32\n\rtrain_lineage\x18\x06 \x01(\x0b\x32\x19.mindinsight.TrainLineageH\x00\x12<\n\x12\x65valuation_lineage\x18\x07 \x01(\x0b\x32\x1e.mindinsight.EvaluationLineageH\x00\x12\x32\n\rdataset_graph\x18\t \x01(\x0b\x32\x19.mindinsight.DatasetGraphH\x00\x42\x06\n\x04what\"\xb6\x04\n\x0cTrainLineage\x12\x43\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32).mindinsight.TrainLineage.HyperParameters\x12=\n\rtrain_dataset\x18\x02 \x01(\x0b\x32&.mindinsight.TrainLineage.TrainDataset\x12\x36\n\talgorithm\x18\x03 \x01(\x0b\x32#.mindinsight.TrainLineage.Algorithm\x12.\n\x05model\x18\x04 \x01(\x0b\x32\x1f.mindinsight.TrainLineage.Model\x1a\xa0\x01\n\x0fHyperParameters\x12\x11\n\toptimizer\x18\x01 \x01(\t\x12\x15\n\rlearning_rate\x18\x02 \x01(\x02\x12\x15\n\rloss_function\x18\x03 \x01(\t\x12\r\n\x05\x65poch\x18\x04 \x01(\x05\x12\x15\n\rparallel_mode\x18\x05 \x01(\t\x12\x12\n\ndevice_num\x18\x06 \x01(\x05\x12\x12\n\nbatch_size\x18\x08 \x01(\x05\x1a\x46\n\x0cTrainDataset\x12\x1a\n\x12train_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12train_dataset_size\x18\x02 \x01(\x05\x1a*\n\tAlgorithm\x12\x0f\n\x07network\x18\x01 \x01(\t\x12\x0c\n\x04loss\x18\x02 \x01(\x02\x1a#\n\x05Model\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\"\xaf\x01\n\x11\x45valuationLineage\x12\x0e\n\x06metric\x18\x02 \x01(\t\x12\x42\n\rvalid_dataset\x18\x03 \x01(\x0b\x32+.mindinsight.EvaluationLineage.ValidDataset\x1a\x46\n\x0cValidDataset\x12\x1a\n\x12valid_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12valid_dataset_size\x18\x02 \x01(\x05\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05value\"\xc4\x01\n\x0c\x44\x61tasetGraph\x12+\n\x08\x63hildren\x18\x01 \x03(\x0b\x32\x19.mindinsight.DatasetGraph\x12\x32\n\tparameter\x18\x02 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12*\n\noperations\x18\x03 \x03(\x0b\x32\x16.mindinsight.Operation\x12\'\n\x07sampler\x18\x04 \x01(\x0b\x32\x16.mindinsight.Operation\"c\n\tOperation\x12\x37\n\x0eoperationParam\x18\x01 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12\x0c\n\x04size\x18\x02 \x03(\x05\x12\x0f\n\x07weights\x18\x03 \x03(\x02\"\xde\x04\n\x12OperationParameter\x12;\n\x06mapStr\x18\x01 \x03(\x0b\x32+.mindinsight.OperationParameter.MapStrEntry\x12\x43\n\nmapStrList\x18\x02 \x03(\x0b\x32/.mindinsight.OperationParameter.MapStrListEntry\x12=\n\x07mapBool\x18\x03 \x03(\x0b\x32,.mindinsight.OperationParameter.MapBoolEntry\x12;\n\x06mapInt\x18\x04 \x03(\x0b\x32+.mindinsight.OperationParameter.MapIntEntry\x12\x41\n\tmapDouble\x18\x05 \x03(\x0b\x32..mindinsight.OperationParameter.MapDoubleEntry\x1a-\n\x0bMapStrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aG\n\x0fMapStrListEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12#\n\x05value\x18\x02 \x01(\x0b\x32\x14.mindinsight.StrList:\x02\x38\x01\x1a.\n\x0cMapBoolEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\x1a-\n\x0bMapIntEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a\x30\n\x0eMapDoubleEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\x1b\n\x07StrList\x12\x10\n\x08strValue\x18\x01 \x03(\tB\x03\xf8\x01\x01' + , + dependencies=[mindinsight__anf__ir__pb2.DESCRIPTOR,]) + + + + +_EVENT = _descriptor.Descriptor( + name='Event', + full_name='mindinsight.Event', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='wall_time', full_name='mindinsight.Event.wall_time', index=0, + number=1, type=1, cpp_type=5, label=2, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='step', full_name='mindinsight.Event.step', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='mindinsight.Event.version', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='graph_def', full_name='mindinsight.Event.graph_def', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='summary', full_name='mindinsight.Event.summary', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='train_lineage', full_name='mindinsight.Event.train_lineage', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='evaluation_lineage', full_name='mindinsight.Event.evaluation_lineage', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='dataset_graph', full_name='mindinsight.Event.dataset_graph', index=7, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='what', full_name='mindinsight.Event.what', + index=0, containing_type=None, fields=[]), + ], + serialized_start=69, + serialized_end=389, +) + + +_TRAINLINEAGE_HYPERPARAMETERS = _descriptor.Descriptor( + name='HyperParameters', + full_name='mindinsight.TrainLineage.HyperParameters', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='optimizer', full_name='mindinsight.TrainLineage.HyperParameters.optimizer', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='learning_rate', full_name='mindinsight.TrainLineage.HyperParameters.learning_rate', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='loss_function', full_name='mindinsight.TrainLineage.HyperParameters.loss_function', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='epoch', full_name='mindinsight.TrainLineage.HyperParameters.epoch', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='parallel_mode', full_name='mindinsight.TrainLineage.HyperParameters.parallel_mode', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_num', full_name='mindinsight.TrainLineage.HyperParameters.device_num', index=5, + number=6, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='batch_size', full_name='mindinsight.TrainLineage.HyperParameters.batch_size', index=6, + number=8, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=645, + serialized_end=805, +) + +_TRAINLINEAGE_TRAINDATASET = _descriptor.Descriptor( + name='TrainDataset', + full_name='mindinsight.TrainLineage.TrainDataset', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='train_dataset_path', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_path', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='train_dataset_size', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_size', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=807, + serialized_end=877, +) + +_TRAINLINEAGE_ALGORITHM = _descriptor.Descriptor( + name='Algorithm', + full_name='mindinsight.TrainLineage.Algorithm', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='network', full_name='mindinsight.TrainLineage.Algorithm.network', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='loss', full_name='mindinsight.TrainLineage.Algorithm.loss', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=879, + serialized_end=921, +) + +_TRAINLINEAGE_MODEL = _descriptor.Descriptor( + name='Model', + full_name='mindinsight.TrainLineage.Model', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='path', full_name='mindinsight.TrainLineage.Model.path', index=0, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='size', full_name='mindinsight.TrainLineage.Model.size', index=1, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=923, + serialized_end=958, +) + +_TRAINLINEAGE = _descriptor.Descriptor( + name='TrainLineage', + full_name='mindinsight.TrainLineage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='hyper_parameters', full_name='mindinsight.TrainLineage.hyper_parameters', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='train_dataset', full_name='mindinsight.TrainLineage.train_dataset', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='algorithm', full_name='mindinsight.TrainLineage.algorithm', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='model', full_name='mindinsight.TrainLineage.model', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_TRAINLINEAGE_HYPERPARAMETERS, _TRAINLINEAGE_TRAINDATASET, _TRAINLINEAGE_ALGORITHM, _TRAINLINEAGE_MODEL, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=392, + serialized_end=958, +) + + +_EVALUATIONLINEAGE_VALIDDATASET = _descriptor.Descriptor( + name='ValidDataset', + full_name='mindinsight.EvaluationLineage.ValidDataset', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='valid_dataset_path', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_path', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='valid_dataset_size', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_size', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1066, + serialized_end=1136, +) + +_EVALUATIONLINEAGE = _descriptor.Descriptor( + name='EvaluationLineage', + full_name='mindinsight.EvaluationLineage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='metric', full_name='mindinsight.EvaluationLineage.metric', index=0, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='valid_dataset', full_name='mindinsight.EvaluationLineage.valid_dataset', index=1, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_EVALUATIONLINEAGE_VALIDDATASET, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=961, + serialized_end=1136, +) + + +_SUMMARY_IMAGE = _descriptor.Descriptor( + name='Image', + full_name='mindinsight.Summary.Image', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='height', full_name='mindinsight.Summary.Image.height', index=0, + number=1, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='width', full_name='mindinsight.Summary.Image.width', index=1, + number=2, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='colorspace', full_name='mindinsight.Summary.Image.colorspace', index=2, + number=3, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='encoded_image', full_name='mindinsight.Summary.Image.encoded_image', index=3, + number=4, type=12, cpp_type=9, label=2, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1193, + serialized_end=1274, +) + +_SUMMARY_VALUE = _descriptor.Descriptor( + name='Value', + full_name='mindinsight.Summary.Value', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='tag', full_name='mindinsight.Summary.Value.tag', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='scalar_value', full_name='mindinsight.Summary.Value.scalar_value', index=1, + number=3, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='image', full_name='mindinsight.Summary.Value.image', index=2, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor', full_name='mindinsight.Summary.Value.tensor', index=3, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='value', full_name='mindinsight.Summary.Value.value', + index=0, containing_type=None, fields=[]), + ], + serialized_start=1277, + serialized_end=1419, +) + +_SUMMARY = _descriptor.Descriptor( + name='Summary', + full_name='mindinsight.Summary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.Summary.value', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1139, + serialized_end=1419, +) + + +_DATASETGRAPH = _descriptor.Descriptor( + name='DatasetGraph', + full_name='mindinsight.DatasetGraph', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='children', full_name='mindinsight.DatasetGraph.children', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='parameter', full_name='mindinsight.DatasetGraph.parameter', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='operations', full_name='mindinsight.DatasetGraph.operations', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sampler', full_name='mindinsight.DatasetGraph.sampler', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1422, + serialized_end=1618, +) + + +_OPERATION = _descriptor.Descriptor( + name='Operation', + full_name='mindinsight.Operation', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='operationParam', full_name='mindinsight.Operation.operationParam', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='size', full_name='mindinsight.Operation.size', index=1, + number=2, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='weights', full_name='mindinsight.Operation.weights', index=2, + number=3, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1620, + serialized_end=1719, +) + + +_OPERATIONPARAMETER_MAPSTRENTRY = _descriptor.Descriptor( + name='MapStrEntry', + full_name='mindinsight.OperationParameter.MapStrEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapStrEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapStrEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2065, + serialized_end=2110, +) + +_OPERATIONPARAMETER_MAPSTRLISTENTRY = _descriptor.Descriptor( + name='MapStrListEntry', + full_name='mindinsight.OperationParameter.MapStrListEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapStrListEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapStrListEntry.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2112, + serialized_end=2183, +) + +_OPERATIONPARAMETER_MAPBOOLENTRY = _descriptor.Descriptor( + name='MapBoolEntry', + full_name='mindinsight.OperationParameter.MapBoolEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapBoolEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapBoolEntry.value', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2185, + serialized_end=2231, +) + +_OPERATIONPARAMETER_MAPINTENTRY = _descriptor.Descriptor( + name='MapIntEntry', + full_name='mindinsight.OperationParameter.MapIntEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapIntEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapIntEntry.value', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2233, + serialized_end=2278, +) + +_OPERATIONPARAMETER_MAPDOUBLEENTRY = _descriptor.Descriptor( + name='MapDoubleEntry', + full_name='mindinsight.OperationParameter.MapDoubleEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapDoubleEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapDoubleEntry.value', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=b'8\001', + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2280, + serialized_end=2328, +) + +_OPERATIONPARAMETER = _descriptor.Descriptor( + name='OperationParameter', + full_name='mindinsight.OperationParameter', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mapStr', full_name='mindinsight.OperationParameter.mapStr', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mapStrList', full_name='mindinsight.OperationParameter.mapStrList', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mapBool', full_name='mindinsight.OperationParameter.mapBool', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mapInt', full_name='mindinsight.OperationParameter.mapInt', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mapDouble', full_name='mindinsight.OperationParameter.mapDouble', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_OPERATIONPARAMETER_MAPSTRENTRY, _OPERATIONPARAMETER_MAPSTRLISTENTRY, _OPERATIONPARAMETER_MAPBOOLENTRY, _OPERATIONPARAMETER_MAPINTENTRY, _OPERATIONPARAMETER_MAPDOUBLEENTRY, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1722, + serialized_end=2328, +) + + +_STRLIST = _descriptor.Descriptor( + name='StrList', + full_name='mindinsight.StrList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='strValue', full_name='mindinsight.StrList.strValue', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2330, + serialized_end=2357, +) + +_EVENT.fields_by_name['graph_def'].message_type = mindinsight__anf__ir__pb2._GRAPHPROTO +_EVENT.fields_by_name['summary'].message_type = _SUMMARY +_EVENT.fields_by_name['train_lineage'].message_type = _TRAINLINEAGE +_EVENT.fields_by_name['evaluation_lineage'].message_type = _EVALUATIONLINEAGE +_EVENT.fields_by_name['dataset_graph'].message_type = _DATASETGRAPH +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['version']) +_EVENT.fields_by_name['version'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['graph_def']) +_EVENT.fields_by_name['graph_def'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['summary']) +_EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['train_lineage']) +_EVENT.fields_by_name['train_lineage'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['evaluation_lineage']) +_EVENT.fields_by_name['evaluation_lineage'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['dataset_graph']) +_EVENT.fields_by_name['dataset_graph'].containing_oneof = _EVENT.oneofs_by_name['what'] +_TRAINLINEAGE_HYPERPARAMETERS.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_TRAINDATASET.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_ALGORITHM.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_MODEL.containing_type = _TRAINLINEAGE +_TRAINLINEAGE.fields_by_name['hyper_parameters'].message_type = _TRAINLINEAGE_HYPERPARAMETERS +_TRAINLINEAGE.fields_by_name['train_dataset'].message_type = _TRAINLINEAGE_TRAINDATASET +_TRAINLINEAGE.fields_by_name['algorithm'].message_type = _TRAINLINEAGE_ALGORITHM +_TRAINLINEAGE.fields_by_name['model'].message_type = _TRAINLINEAGE_MODEL +_EVALUATIONLINEAGE_VALIDDATASET.containing_type = _EVALUATIONLINEAGE +_EVALUATIONLINEAGE.fields_by_name['valid_dataset'].message_type = _EVALUATIONLINEAGE_VALIDDATASET +_SUMMARY_IMAGE.containing_type = _SUMMARY +_SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE +_SUMMARY_VALUE.fields_by_name['tensor'].message_type = mindinsight__anf__ir__pb2._TENSORPROTO +_SUMMARY_VALUE.containing_type = _SUMMARY +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['scalar_value']) +_SUMMARY_VALUE.fields_by_name['scalar_value'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['image']) +_SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['tensor']) +_SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE +_DATASETGRAPH.fields_by_name['children'].message_type = _DATASETGRAPH +_DATASETGRAPH.fields_by_name['parameter'].message_type = _OPERATIONPARAMETER +_DATASETGRAPH.fields_by_name['operations'].message_type = _OPERATION +_DATASETGRAPH.fields_by_name['sampler'].message_type = _OPERATION +_OPERATION.fields_by_name['operationParam'].message_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPSTRENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPSTRLISTENTRY.fields_by_name['value'].message_type = _STRLIST +_OPERATIONPARAMETER_MAPSTRLISTENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPBOOLENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPINTENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPDOUBLEENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER.fields_by_name['mapStr'].message_type = _OPERATIONPARAMETER_MAPSTRENTRY +_OPERATIONPARAMETER.fields_by_name['mapStrList'].message_type = _OPERATIONPARAMETER_MAPSTRLISTENTRY +_OPERATIONPARAMETER.fields_by_name['mapBool'].message_type = _OPERATIONPARAMETER_MAPBOOLENTRY +_OPERATIONPARAMETER.fields_by_name['mapInt'].message_type = _OPERATIONPARAMETER_MAPINTENTRY +_OPERATIONPARAMETER.fields_by_name['mapDouble'].message_type = _OPERATIONPARAMETER_MAPDOUBLEENTRY +DESCRIPTOR.message_types_by_name['Event'] = _EVENT +DESCRIPTOR.message_types_by_name['TrainLineage'] = _TRAINLINEAGE +DESCRIPTOR.message_types_by_name['EvaluationLineage'] = _EVALUATIONLINEAGE +DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY +DESCRIPTOR.message_types_by_name['DatasetGraph'] = _DATASETGRAPH +DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION +DESCRIPTOR.message_types_by_name['OperationParameter'] = _OPERATIONPARAMETER +DESCRIPTOR.message_types_by_name['StrList'] = _STRLIST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), { + 'DESCRIPTOR' : _EVENT, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Event) + }) +_sym_db.RegisterMessage(Event) + +TrainLineage = _reflection.GeneratedProtocolMessageType('TrainLineage', (_message.Message,), { + + 'HyperParameters' : _reflection.GeneratedProtocolMessageType('HyperParameters', (_message.Message,), { + 'DESCRIPTOR' : _TRAINLINEAGE_HYPERPARAMETERS, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.HyperParameters) + }) + , + + 'TrainDataset' : _reflection.GeneratedProtocolMessageType('TrainDataset', (_message.Message,), { + 'DESCRIPTOR' : _TRAINLINEAGE_TRAINDATASET, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.TrainDataset) + }) + , + + 'Algorithm' : _reflection.GeneratedProtocolMessageType('Algorithm', (_message.Message,), { + 'DESCRIPTOR' : _TRAINLINEAGE_ALGORITHM, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Algorithm) + }) + , + + 'Model' : _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), { + 'DESCRIPTOR' : _TRAINLINEAGE_MODEL, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Model) + }) + , + 'DESCRIPTOR' : _TRAINLINEAGE, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage) + }) +_sym_db.RegisterMessage(TrainLineage) +_sym_db.RegisterMessage(TrainLineage.HyperParameters) +_sym_db.RegisterMessage(TrainLineage.TrainDataset) +_sym_db.RegisterMessage(TrainLineage.Algorithm) +_sym_db.RegisterMessage(TrainLineage.Model) + +EvaluationLineage = _reflection.GeneratedProtocolMessageType('EvaluationLineage', (_message.Message,), { + + 'ValidDataset' : _reflection.GeneratedProtocolMessageType('ValidDataset', (_message.Message,), { + 'DESCRIPTOR' : _EVALUATIONLINEAGE_VALIDDATASET, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage.ValidDataset) + }) + , + 'DESCRIPTOR' : _EVALUATIONLINEAGE, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage) + }) +_sym_db.RegisterMessage(EvaluationLineage) +_sym_db.RegisterMessage(EvaluationLineage.ValidDataset) + +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { + + 'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_IMAGE, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Summary.Image) + }) + , + + 'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_VALUE, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Summary.Value) + }) + , + 'DESCRIPTOR' : _SUMMARY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Summary) + }) +_sym_db.RegisterMessage(Summary) +_sym_db.RegisterMessage(Summary.Image) +_sym_db.RegisterMessage(Summary.Value) + +DatasetGraph = _reflection.GeneratedProtocolMessageType('DatasetGraph', (_message.Message,), { + 'DESCRIPTOR' : _DATASETGRAPH, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.DatasetGraph) + }) +_sym_db.RegisterMessage(DatasetGraph) + +Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), { + 'DESCRIPTOR' : _OPERATION, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Operation) + }) +_sym_db.RegisterMessage(Operation) + +OperationParameter = _reflection.GeneratedProtocolMessageType('OperationParameter', (_message.Message,), { + + 'MapStrEntry' : _reflection.GeneratedProtocolMessageType('MapStrEntry', (_message.Message,), { + 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPSTRENTRY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrEntry) + }) + , + + 'MapStrListEntry' : _reflection.GeneratedProtocolMessageType('MapStrListEntry', (_message.Message,), { + 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPSTRLISTENTRY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrListEntry) + }) + , + + 'MapBoolEntry' : _reflection.GeneratedProtocolMessageType('MapBoolEntry', (_message.Message,), { + 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPBOOLENTRY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapBoolEntry) + }) + , + + 'MapIntEntry' : _reflection.GeneratedProtocolMessageType('MapIntEntry', (_message.Message,), { + 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPINTENTRY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapIntEntry) + }) + , + + 'MapDoubleEntry' : _reflection.GeneratedProtocolMessageType('MapDoubleEntry', (_message.Message,), { + 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPDOUBLEENTRY, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapDoubleEntry) + }) + , + 'DESCRIPTOR' : _OPERATIONPARAMETER, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter) + }) +_sym_db.RegisterMessage(OperationParameter) +_sym_db.RegisterMessage(OperationParameter.MapStrEntry) +_sym_db.RegisterMessage(OperationParameter.MapStrListEntry) +_sym_db.RegisterMessage(OperationParameter.MapBoolEntry) +_sym_db.RegisterMessage(OperationParameter.MapIntEntry) +_sym_db.RegisterMessage(OperationParameter.MapDoubleEntry) + +StrList = _reflection.GeneratedProtocolMessageType('StrList', (_message.Message,), { + 'DESCRIPTOR' : _STRLIST, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.StrList) + }) +_sym_db.RegisterMessage(StrList) + + +DESCRIPTOR._options = None +_OPERATIONPARAMETER_MAPSTRENTRY._options = None +_OPERATIONPARAMETER_MAPSTRLISTENTRY._options = None +_OPERATIONPARAMETER_MAPBOOLENTRY._options = None +_OPERATIONPARAMETER_MAPINTENTRY._options = None +_OPERATIONPARAMETER_MAPDOUBLEENTRY._options = None +# @@protoc_insertion_point(module_scope) diff --git a/mindinsight/datavisual/utils/__init__.py b/mindinsight/datavisual/utils/__init__.py new file mode 100644 index 00000000..18521a26 --- /dev/null +++ b/mindinsight/datavisual/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Utils.""" +from .tools import find_app_package diff --git a/mindinsight/datavisual/utils/crc32/base.h b/mindinsight/datavisual/utils/crc32/base.h new file mode 100644 index 00000000..c9bf749b --- /dev/null +++ b/mindinsight/datavisual/utils/crc32/base.h @@ -0,0 +1,57 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATAVISUAL_UTILS_CRC32_BASE_H_ +#define DATAVISUAL_UTILS_CRC32_BASE_H_ + +#include +#include +#include "securec/include/securec.h" + +using string = std::string; + +using int8 = int8_t; +using int16 = int16_t; +using int32 = int32_t; +using int64 = int64_t; + +using uint8 = uint8_t; +using uint16 = uint16_t; +using uint32 = uint32_t; +using uint64 = uint64_t; + +// check the null point, Only log it in if(): The value is null +#define EXCEPT_CHECK_NULL(value) \ + do { \ + if (value == nullptr) { \ + break; \ + } \ + } while (0) + +// implement common define function +// Get the 32 bits align value +inline uint32 DecodeFixed32(const char* ptr) { + uint32 result = 0; + if (EOK != memcpy_s(&result, sizeof(result), ptr, sizeof(result))) { + return result; + } + return result; +} + +// Used to fetch a naturally-aligned 32-bit word in little endian byte-order +inline uint32 LE_LOAD32(const uint8_t* p) { return DecodeFixed32(reinterpret_cast(p)); } + +#endif // DATAVISUAL_UTILS_CRC32_BASE_H_ diff --git a/mindinsight/datavisual/utils/crc32/crc32.cc b/mindinsight/datavisual/utils/crc32/crc32.cc new file mode 100644 index 00000000..6c0f0289 --- /dev/null +++ b/mindinsight/datavisual/utils/crc32/crc32.cc @@ -0,0 +1,316 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "crc32/crc32.h" +#include + +const unsigned int CRC_TABLE_SIZE = 256; + +static const uint32 crc_table_o32[CRC_TABLE_SIZE] = { + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, + 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, + 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, + 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, + 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, + 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, + 0x6EF07595, 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, 0x5125DAD3, + 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, + 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, 0x61C69362, 0x93AD1061, 0x80FDE395, + 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, + 0x44694011, 0x5739B3E5, 0xA55230E6, 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, + 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, + 0x563C5F93, 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, 0x1871A4D8, + 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5, + 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, + 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, + 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, + 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, + 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, + 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, + 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, 0xF36E6F75, 0x0105EC76, 0x12551F82, + 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351}; + +static const uint32 crc_table_o40[CRC_TABLE_SIZE] = { + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, 0x9D14C3B8, + 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, 0x3FC5F181, 0x2C6769F6, + 0x1880C16F, 0x0B225918, 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, 0xA2D13239, 0xB173AA4E, 0x859402D7, + 0x96369AA0, 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, + 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, 0xAC154166, + 0xBFB7D911, 0x8B507188, 0x98F2E9FF, 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, + 0x298143B1, 0x3A23DBC6, 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, 0x93D0B0E7, 0x80722890, 0xB4958009, + 0xA737187E, 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, 0xC0D23785, + 0xD370AFF2, 0xE797076B, 0xF4359F1C, 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, 0x5DC6F43D, 0x4E646C4A, + 0x7A83C4D3, 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, 0x809C2506, 0x933EBD71, 0xA7D915E8, + 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, + 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, + 0xE2712D2C, 0xD69685B5, 0xC5341DC2, 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, + 0x4B82460D, 0x5820DE7A, 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, 0xB5499B25, 0xA6EB0352, 0x920CABCB, + 0x81AE33BC, 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, 0x5912C8C0, + 0x4AB050B7, 0x7E57F82E, 0x6DF56059, 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, 0x844819FB, 0x97EA818C, + 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, + 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, + 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, 0x68134A1E, + 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, 0x4A5E5D21, 0x59FCC556, + 0x6D1B6DCF, 0x7EB9F5B8, 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, + 0xE3AD3600, 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, 0x7B5FDFFF, + 0x68FD4788, 0x5C1AEF11, 0x4FB87766, 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, 0xE64B1C47, 0xF5E98430, + 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, 0x449A2E7E, 0x5738B609, 0x63DF1E90, + 0x707D86E7, 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, + 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483}; + +static const uint32 crc_table_o48[CRC_TABLE_SIZE] = { + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, 0x38513EC5, + 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, 0x70A27D8A, 0xD5E3EFF4, + 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, 0x48F3434F, 0xEDB2D131, 0x079C1142, + 0xA2DD833C, 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, + 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, 0x47CB61CB, + 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, + 0x40577089, 0xE516E2F7, 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, 0x37691C41, 0x92288E3F, 0x78064E4C, + 0xDD47DC32, 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, 0xB7C7FD53, + 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, 0x8F96C396, 0x2AD751E8, + 0xC0F9919B, 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, 0x26217BCD, 0x8360E9B3, 0x694E29C0, + 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, + 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, 0xC85DA25D, + 0x6D1C3023, 0x8732F050, 0x2273622E, 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, + 0xBF63CE95, 0x1A225CEB, 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, 0x15F9D359, 0xB0B84127, 0x5A968154, + 0xFFD7132A, 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, 0xC3D4340C, + 0x6695A672, 0x8CBB6601, 0x29FAF47F, 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, 0x6A638C57, 0xCF221E29, + 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, 0x5232B292, 0xF77320EC, 0x1D5DE09F, + 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, + 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, + 0x190FF97C, 0xF321390F, 0x5660AB71, 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, 0xD29C5380, 0x77DDC1FE, + 0x9DF3018D, 0x38B293F3, 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, + 0x00E3AD36, 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, 0xAD060C8E, + 0x08479EF0, 0xE2695E83, 0x4728CCFD, 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, 0x9557324B, 0x3016A035, + 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, 0xDDA47104, 0x78E5E37A, 0x92CB2309, + 0x378AB177, 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, + 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8}; + +static const uint32 crc_table_o56[CRC_TABLE_SIZE] = { + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, 0xF64463E6, + 0x2B01C95E, 0x49234067, 0x9466EADF, 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, 0xE964B13D, 0x34211B85, + 0x560392BC, 0x8B463804, 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, 0x1F20D2DB, 0xC2657863, 0xA047F15A, + 0x7D025BE2, 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, + 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, 0x5A43469E, + 0x8706EC26, 0xE524651F, 0x3861CFA7, 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, + 0xFA04B7C4, 0x27411D7C, 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, 0xB327F7A3, 0x6E625D1B, 0x0C40D422, + 0xD1057E9A, 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, 0x42C2EEDA, + 0x9F874462, 0xFDA5CD5B, 0x20E067E3, 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, 0xB4868D3C, 0x69C32784, + 0x0BE1AEBD, 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, + 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, + 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, + 0x3380611A, 0x51A2E823, 0x8CE7429B, 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, + 0xA7E68BC5, 0x7AA3217D, 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, + 0x4BA071F5, 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, 0x4D801BE4, + 0x90C5B15C, 0xF2E73865, 0x2FA292DD, 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, 0x8585DDB4, 0x58C0770C, + 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, + 0x11E3376B, 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, + 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, 0xE1873E9C, + 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, 0x8224A72B, 0x5F610D93, + 0x3D4384AA, 0xE0062E12, 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, 0x7460C4CD, 0xA9256E75, 0xCB07E74C, + 0x16424DF4, 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, 0x2E238253, + 0xF36628EB, 0x9144A1D2, 0x4C010B6A, 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, 0xD867E1B5, 0x05224B0D, + 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, 0xC747336E, 0x1A0299D6, 0x782010EF, + 0xA565BA57, 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, + 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842}; + +static const uint32 crc_table_o64[CRC_TABLE_SIZE] = { + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, 0xC5670B91, + 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, 0x8F2261D3, 0xB7330E7F, + 0xFF00BE8B, 0xC711D127, 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, 0x4A456A42, 0x725405EE, 0x3A67B51A, + 0x0276DAB6, 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, + 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, 0x3E8A0076, + 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, + 0x04EDB56C, 0x3CFCDAC0, 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, + 0xF99BD151, 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, 0xB8730B7D, + 0x806264D1, 0xC851D425, 0xF040BB89, 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, 0x7D1400EC, 0x45056F40, + 0x0D36DFB4, 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, + 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, + 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, 0x439E009A, + 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, + 0xF6DBD453, 0xCECABBFF, 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, + 0xC6D4DB18, 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, 0x24E7BF1E, + 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, 0x750A600B, 0x4D1B0FA7, + 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, + 0xF85EDB6E, 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, + 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, + 0xE71BDB55, 0xAF286BA1, 0x9739040D, 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, 0xB9B60142, 0x81A76EEE, + 0xC994DE1A, 0xF185B1B6, 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, + 0x34E2BA27, 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, 0x425B0AA5, + 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, 0x873C0134, 0xBF2D6E98, + 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, 0xCD796B76, 0xF56804DA, 0xBD5BB42E, + 0x854ADB82, 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, + 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3}; + +static const uint32 crc_table_o72[CRC_TABLE_SIZE] = { + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, 0x6006181F, + 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, 0xC00C303E, 0x2F3C5B27, + 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, 0xA00A2821, 0x4F3A4338, 0x7B8688E2, + 0x94B6E3FB, 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, + 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, 0x570739E5, + 0xB83752FC, 0x8C8B9926, 0x63BBF23F, 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, + 0x2C81B107, 0xC3B1DA1E, 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, 0x970B09DB, 0x783B62C2, 0x4C87A918, + 0xA3B7C201, 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, 0xCE086BD5, + 0x213800CC, 0x1584CB16, 0xFAB4A00F, 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, 0xAE0E73CA, 0x413E18D3, + 0x7582D309, 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, 0x8BF04D66, 0x64C0267F, 0x507CEDA5, + 0xBF4C86BC, 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, + 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, 0xF9094A2F, + 0x16392136, 0x2285EAEC, 0xCDB581F5, 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, + 0x4283F2F3, 0xADB399EA, 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, 0xAEFD80A1, 0x41CDEBB8, 0x75712062, + 0x9A414B7B, 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, 0xBC029FF7, + 0x5332F4EE, 0x678E3F34, 0x88BE542D, 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, 0x99FCA15B, 0x76CCCA42, + 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, 0xF9FAB944, 0x16CAD25D, 0x22761987, + 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, + 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, + 0x6433D514, 0x508F1ECE, 0xBFBF75D7, 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, 0xA0F9DB4A, 0x4FC9B053, + 0x7B757B89, 0x94451090, 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, 0xC0FFC355, 0x2FCFA84C, 0x1B736396, + 0xF443088F, 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, 0x97F8FAB0, + 0x78C891A9, 0x4C745A73, 0xA344316A, 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, 0xF7FEE2AF, 0x18CE89B6, + 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, + 0x63480154, 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, + 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C}; + +static const uint32 crc_table_o80[CRC_TABLE_SIZE] = { + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, 0x4E2DFD53, + 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, 0x9C5BFAA6, 0xF458D66E, + 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, 0xD27607F5, 0xBA752B3D, 0x02705E65, + 0x6A7372AD, 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, + 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, 0xD696BB3F, + 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, + 0xD4E6E55A, 0xBCE5C992, 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, + 0xF2C834C1, 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, 0xE6ECFDDC, + 0x8EEFD114, 0x36EAA44C, 0x5EE98884, 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, 0xA8C1008F, 0xC0C22C47, + 0x78C7591F, 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, + 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, + 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, + 0x16549778, 0xAE51E220, 0xC652CEE8, 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, + 0xE07C1F73, 0x887F33BB, 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, 0x508ECB25, 0x388DE7ED, 0x808892B5, + 0xE88BBE7D, 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, 0x27180901, + 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, 0xC8358D49, 0xA036A181, + 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, 0x8618701A, 0xEE1B5CD2, 0x561E298A, + 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, + 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, + 0xD7A063A5, 0x6FA516FD, 0x07A63A35, 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, 0x2A39CC5F, 0x423AE097, + 0xFA3F95CF, 0x923CB907, 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, 0x6414310C, 0x0C171DC4, 0xB412689C, + 0xDC114454, 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, 0xB2828A33, + 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, 0xFCAF7760, 0x94AC5BA8, + 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, + 0x96DC05CD, 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, + 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F}; + +static const uint32 crc_table_o88[CRC_TABLE_SIZE] = { + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, 0x423B04DA, + 0x0B0779FD, 0xD043FE94, 0x997F83B3, 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, 0x847609B4, 0xCD4A7493, + 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, 0xC64D0D6E, 0x8F717049, 0x5435F720, + 0x1D098A07, 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, + 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, 0x6E26E32E, + 0x271A9E09, 0xFC5E1960, 0xB5626447, 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, + 0x3A13140E, 0x732F6929, 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, 0xEA50EA9A, 0xA36C97BD, 0x782810D4, + 0x31146DF3, 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, 0x9E76C286, + 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, 0xDC4DC65C, 0x9571BB7B, + 0x4E353C12, 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, 0x1700AEAB, 0x5E3CD38C, 0x857854E5, + 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, + 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, 0xB26B2572, + 0xFB575855, 0x2013DF3C, 0x692FA21B, 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, + 0x6228DBE6, 0x2B14A6C1, 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, 0x151C1409, 0x5C20692E, 0x8764EE47, + 0xCE589360, 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, 0xF24C9B0A, + 0xBB70E62D, 0x60346144, 0x29081C63, 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, 0x3901F3FD, 0x703D8EDA, + 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, 0x7B3AF727, 0x32068A00, 0xE9420D69, + 0xA07E704E, 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, + 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, 0xDE517CFE, + 0x976D01D9, 0x4C2986B0, 0x0515FB97, 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, 0x0F1CDF3B, 0x4620A21C, + 0x9D642575, 0xD4585852, 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, + 0x96635C88, 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, 0x230138CF, + 0x6A3D45E8, 0xB179C281, 0xF845BFA6, 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, 0x613A3C15, 0x28064132, + 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, 0xA777317B, 0xEE4B4C5C, 0x350FCB35, + 0x7C33B612, 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, + 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5}; + +// Use the 8 table to calc crc32c value +inline void CRC32T8(uint32 *crc, const uint8_t **p) { + auto c = static_cast(*crc ^ LE_LOAD32(*p)); + *p += 4; + *crc = crc_table_o88[c & 0xff] ^ crc_table_o80[(c >> 8) & 0xff] ^ crc_table_o72[(c >> 16) & 0xff] ^ + crc_table_o64[(c >> 24) & 0xff]; + c = static_cast(LE_LOAD32(*p)); + *crc = (*crc) ^ crc_table_o56[c & 0xff] ^ crc_table_o48[(c >> 8) & 0xff] ^ crc_table_o40[(c >> 16) & 0xff] ^ + crc_table_o32[(c >> 24) & 0xff]; + *p += 4; +} + +// calc the crc32c value +uint32 MakeCrc32c(uint32 init_crc, const char *data, size_t size) { + EXCEPT_CHECK_NULL(data); + uint32 crc = init_crc ^ 0xffffffffu; + const unsigned int OFFSET = 8; + + // Get the origin begin and end address(not alignment) + auto *bp = reinterpret_cast(data); + const uint8_t *ep = bp + size; + + // Get the alignment address + // Point x at first 4-byte aligned byte in string. + // This might be just past the end of the string. + auto pval = reinterpret_cast(bp); + auto *bp_align = reinterpret_cast(MEM_ALIGN(pval, 2)); + + // process the not alignment bits when size < 4 byte + if (bp_align <= ep) { + // Process bytes until finished or p is 4-byte aligned + while (bp != bp_align) { + crc = crc_table_o32[(crc & 0xff) ^ (*bp++)] ^ (crc >> 8); + } + } + + // Process bytes 8 at a time use the 8 table + while ((ep - bp) >= OFFSET) { + CRC32T8(&crc, &bp); + } + + // Process the last not alignment bytes + while (bp < ep) { + crc = crc_table_o32[(crc & 0xff) ^ (*bp++)] ^ (crc >> 8); + } + return crc ^ 0xffffffffu; +} diff --git a/mindinsight/datavisual/utils/crc32/crc32.h b/mindinsight/datavisual/utils/crc32/crc32.h new file mode 100644 index 00000000..80c0ce60 --- /dev/null +++ b/mindinsight/datavisual/utils/crc32/crc32.h @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATAVISUAL_UTILS_CRC32_CRC32_H_ +#define DATAVISUAL_UTILS_CRC32_CRC32_H_ + +#include +#include +#include +#include "crc32/base.h" + +// Align n to (1 << m) byte boundary +#define MEM_ALIGN(n, m) ((n + ((1 << m) - 1)) & ~((1 << m) - 1)) + +// Masked for crc. +static constexpr uint32 kMaskDelta = 0xa282ead8ul; + +// Provide the Crc32c function + +// Calculate the crc32c value, use the 8 table method +uint32 MakeCrc32c(uint32 init_crc, const char* data, size_t size); + +uint32 GetMaskCrc32cValue(const char* data, size_t n) { + auto crc = MakeCrc32c(0, data, n); + return crc; +} + +uint32 GetValueFromStr(const char* crc_str) { + uint32 crc = DecodeFixed32(crc_str); + uint32 rot = crc - kMaskDelta; + return ((rot >> 17) | (rot << 15)); +} + +PYBIND11_MODULE(crc32, m) { + m.doc() = "crc util"; + m.def("MakeCrc32c", &MakeCrc32c, "A function calculating the crc32c value, use the 8 table method"); + m.def("GetMaskCrc32cValue", &GetMaskCrc32cValue, "A function return the crc32c value"); + m.def("GetValueFromStr", &GetValueFromStr, "A function return the crc32c value from string"); +} + +#endif // DATAVISUAL_UTILS_CRC32_CRC32_H_ diff --git a/mindinsight/datavisual/utils/tools.py b/mindinsight/datavisual/utils/tools.py new file mode 100644 index 00000000..806132ae --- /dev/null +++ b/mindinsight/datavisual/utils/tools.py @@ -0,0 +1,155 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Common Tools.""" +import imghdr +import math +import os + +from numbers import Number +from urllib.parse import unquote + +from mindinsight.utils import exceptions + +_IMG_EXT_TO_MIMETYPE = { + 'bmp': 'image/bmp', + 'gif': 'image/gif', + 'jpeg': 'image/jpeg', + 'png': 'image/png', +} +_DEFAULT_IMAGE_MIMETYPE = 'application/octet-stream' + + +def find_app_package(): + """Find package in current directory.""" + backend_dir = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, os.pardir, "backend")) + packages = [] + for file in os.listdir(backend_dir): + file_path = os.path.join(backend_dir, file) + if os.path.isfile(file_path): + continue + if not os.path.isfile(os.path.join(file_path, '__init__.py')): + continue + rel_path = os.path.relpath(file_path, backend_dir) + package = rel_path.replace(os.path.sep, '.') + package = f"mindinsight.backend.{package}" + packages.append(package) + return packages + + +def to_str(bytes_or_text, encode="utf-8"): + """Bytes transform string.""" + if isinstance(bytes_or_text, bytes): + return bytes_or_text.decode(encode) + if isinstance(bytes_or_text, str): + return bytes_or_text + + raise TypeError("Param isn't str or bytes type, param={}".format(bytes_or_text)) + + +def to_int(param, param_name): + """ + Transfer param to int type. + + Args: + param (Any): A param transformed. + param_name (str): Param name. + + Returns: + int, value after transformed. + + """ + try: + param = int(param) + except ValueError: + raise exceptions.ParamTypeError(param_name, 'Integer') + return param + + +def str_to_bool(param, param_name): + """ + Check param and transform it to bool. + + Args: + param (str): 'true' or 'false' is valid. + param_name (str): Param name. + + Returns: + bool, if param is 'true', case insensitive. + + Raises: + ParamValueError: If the value of param is not 'false' and 'true'. + + """ + if not isinstance(param, str): + raise exceptions.ParamTypeError(param_name, 'str') + + if param.lower() not in ['false', 'true']: + raise exceptions.ParamValueError("The value of %s must be 'false' or 'true'." % param_name) + param = (param.lower() == 'true') + + return param + + +def get_img_mimetype(img_data): + """ + Recognize image headers and generate image MIMETYPE. + + Args: + img_data (bin): Binary character stream of image. + + Returns: + str, a MIMETYPE of the give image. + """ + image_type = imghdr.what(None, img_data) + mimetype = _IMG_EXT_TO_MIMETYPE.get(image_type, _DEFAULT_IMAGE_MIMETYPE) + return mimetype + + +def get_train_id(request): + """ + Get train ID from requst query string and unquote content. + + Args: + request (FlaskRequest): Http request instance. + + Returns: + str, unquoted train ID. + """ + train_id = request.args.get('train_id') + if train_id is not None: + try: + train_id = unquote(train_id, errors='strict') + except UnicodeDecodeError: + raise exceptions.ParamValueError('Unquote error with strict mode') + return train_id + + +def if_nan_inf_to_none(name, value): + """ + Transform value to None if it is NaN or Inf. + + Args: + name (str): Name of value. + value (float): A number transformed. + + Returns: + float, if value is NaN or Inf, return None. + + """ + if not isinstance(value, Number): + raise exceptions.ParamTypeError(name, 'number') + if math.isnan(value) or math.isinf(value): + value = None + return value diff --git a/mindinsight/lineagemgr/__init__.py b/mindinsight/lineagemgr/__init__.py new file mode 100644 index 00000000..d89c3c1d --- /dev/null +++ b/mindinsight/lineagemgr/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Lineagemgr Module Introduction. + +This module provides Python APIs to collect and query the lineage of models. +Users can add the TrainLineage/EvalLineage callback to the MindSpore train/eval callback list to +collect the key parameters and results, such as, the name of the network and optimizer, the +evaluation metric and results. +The APIs can be used to get the lineage information of the models. For example, +what hyperparameter is used in the model training, which model has the highest +accuracy among all the versions, etc. +""" +from mindinsight.lineagemgr.api.model import get_summary_lineage, filter_summary_lineage +from mindinsight.lineagemgr.common.log import logger +try: + from mindinsight.lineagemgr.collection.model.model_lineage import TrainLineage, EvalLineage +except (ModuleNotFoundError, NameError, ImportError): + logger.warning('Not found MindSpore!') + +__all__ = ["TrainLineage", "EvalLineage", "get_summary_lineage", "filter_summary_lineage"] diff --git a/mindinsight/lineagemgr/api/__init__.py b/mindinsight/lineagemgr/api/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/api/model.py b/mindinsight/lineagemgr/api/model.py new file mode 100644 index 00000000..b6847201 --- /dev/null +++ b/mindinsight/lineagemgr/api/model.py @@ -0,0 +1,292 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file is used to define the model lineage python api.""" +import os + +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamValueError, \ + LineageFileNotFoundError, LineageQuerySummaryDataError, LineageParamSummaryPathError, \ + LineageQuerierParamException, LineageDirNotExistError, LineageSearchConditionParamError, \ + LineageParamTypeError, LineageSummaryParseException +from mindinsight.lineagemgr.common.log import logger as log +from mindinsight.lineagemgr.common.path_parser import SummaryPathParser +from mindinsight.lineagemgr.common.validator.model_parameter import SearchModelConditionParameter +from mindinsight.lineagemgr.common.validator.validate import validate_filter_key +from mindinsight.lineagemgr.common.validator.validate import validate_search_model_condition, \ + validate_condition, validate_path +from mindinsight.lineagemgr.querier.querier import Querier +from mindinsight.utils.exceptions import MindInsightException + + +def get_summary_lineage(summary_dir, keys=None): + """ + Get the lineage information according to summary directory and keys. + + The function queries lineage information of single train process + corresponding to the given summary directory. Users can query the + information according to `keys`. + + Args: + summary_dir (str): The summary directory. It contains summary logs for + one training. + keys (list[str]): The filter keys of lineage information. The acceptable + keys are `metric`, `hyper_parameters`, `algorithm`, `train_dataset`, + `model`, `valid_dataset` and `dataset_graph`. If it is `None`, all + information will be returned. Default: None. + + Returns: + dict, the lineage information for one training. + + Raises: + LineageParamSummaryPathError: If summary path is invalid. + LineageQuerySummaryDataError: If querying summary data fails. + LineageFileNotFoundError: If the summary log file is not found. + + Examples: + >>> summary_dir = "/path/to/summary" + >>> summary_lineage_info = get_summary_lineage(summary_dir) + >>> hyper_parameters = get_summary_lineage(summary_dir, keys=["hyper_parameters"]) + """ + try: + summary_dir = validate_path(summary_dir) + except MindInsightException as error: + log.error(str(error)) + log.exception(error) + raise LineageParamSummaryPathError(str(error.message)) + + if keys is not None: + validate_filter_key(keys) + + summary_path = SummaryPathParser.get_latest_lineage_summary(summary_dir) + if summary_path is None: + log.error('There is no summary log file under summary_dir.') + raise LineageFileNotFoundError( + 'There is no summary log file under summary_dir.' + ) + + try: + result = Querier(summary_path).get_summary_lineage( + summary_dir, filter_keys=keys) + except LineageSummaryParseException: + return {} + except (LineageQuerierParamException, LineageParamTypeError) as error: + log.error(str(error)) + log.exception(error) + raise LineageQuerySummaryDataError("Get summary lineage failed.") + + return result[0] + + +def filter_summary_lineage(summary_base_dir, search_condition=None): + """ + Filter the lineage information under summary base directory according to search condition. + + Users can filter and sort all lineage information according to the search + condition. The supported filter fields include `summary_dir`, `network`, + etc. The filter conditions include `eq`, `lt`, `gt`, `le`, `ge` and `in`. + At the same time, the combined use of these fields and conditions is + supported. If you want to sort based on filter fields, the field of + `sorted_name` and `sorted_type` should be specified. + + Users can use `lineage_type` to decide what kind of lineage information to + query. If the `lineage_type` is `dataset`, the query result is only the + lineage information related to data augmentation. If the `lineage_type` is + `model` or `None`, the query result is all lineage information. + + Users can paginate query result based on `offset` and `limit`. The `offset` + refers to page number. The `limit` refers to the number in one page. + + Args: + summary_base_dir (str): The summary base directory. It contains summary + directories generated by training. + search_condition (dict): The search condition. When filtering and + sorting, in addition to the following supported fields, fields + prefixed with `metric_` are also supported. The fields prefixed with + `metric_` are related to the `metrics` parameter in the training + script. For example, if the key of `metrics` parameter is + `accuracy`, the field should be `metric_accuracy`. Default: None. + + - summary_dir (dict): The filter condition of summary directory. + + - loss_function (dict): The filter condition of loss function. + + - train_dataset_path (dict): The filter condition of train dataset path. + + - train_dataset_count (dict): The filter condition of train dataset count. + + - test_dataset_path (dict): The filter condition of test dataset path. + + - test_dataset_count (dict): The filter condition of test dataset count. + + - network (dict): The filter condition of network. + + - optimizer (dict): The filter condition of optimizer. + + - learning_rate (dict): The filter condition of learning rate. + + - epoch (dict): The filter condition of epoch. + + - batch_size (dict): The filter condition of batch size. + + - loss (dict): The filter condition of loss. + + - model_size (dict): The filter condition of model size. + + - dataset_mark (dict): The filter condition of dataset mark. + + - offset (int): Page number, the value range is [0, 100000]. + + - limit (int): The number in one page, the value range is [1, 100]. + + - sorted_name (str): Specify which field to sort by. + + - sorted_type (str): Specify sort order. It can be `ascending` or + `descending`. + + - lineage_type (str): It decides what kind of lineage information to + query. It can be `dataset` or `model`. If it is `dataset`, + the query result is only the lineage information related to data + augmentation. If it is `model` or `None`, the query result is all + lineage information. + + Returns: + dict, all lineage information under summary base directory according to + search condition. + + Raises: + LineageSearchConditionParamError: If search_condition param is invalid. + LineageParamSummaryPathError: If summary path is invalid. + LineageFileNotFoundError: If the summary log file is not found. + LineageQuerySummaryDataError: If querying summary log file data fails. + + Examples: + >>> summary_base_dir = "/path/to/summary_base" + >>> search_condition = { + >>> 'summary_dir': { + >>> 'in': [ + >>> os.path.join(summary_base_dir, 'summary_1'), + >>> os.path.join(summary_base_dir, 'summary_2'), + >>> os.path.join(summary_base_dir, 'summary_3') + >>> ] + >>> }, + >>> 'loss': { + >>> 'gt': 2.0 + >>> }, + >>> 'batch_size': { + >>> 'ge': 128, + >>> 'le': 256 + >>> }, + >>> 'metric_accuracy': { + >>> 'lt': 0.1 + >>> }, + >>> 'sorted_name': 'summary_dir', + >>> 'sorted_type': 'descending', + >>> 'limit': 3, + >>> 'offset': 0, + >>> 'lineage_type': 'model' + >>> } + >>> summary_lineage = filter_summary_lineage(summary_base_dir) + >>> summary_lineage_filter = filter_summary_lineage(summary_base_dir, search_condition) + """ + try: + summary_base_dir = validate_path(summary_base_dir) + except (LineageParamValueError, LineageDirNotExistError) as error: + log.error(str(error)) + log.exception(error) + raise LineageParamSummaryPathError(str(error.message)) + + search_condition = {} if search_condition is None else search_condition + + try: + validate_condition(search_condition) + validate_search_model_condition(SearchModelConditionParameter, search_condition) + except MindInsightException as error: + log.error(str(error)) + log.exception(error) + raise LineageSearchConditionParamError(str(error.message)) + + try: + search_condition = _convert_relative_path_to_abspath(summary_base_dir, search_condition) + except (LineageParamValueError, LineageDirNotExistError) as error: + log.error(str(error)) + log.exception(error) + raise LineageParamSummaryPathError(str(error.message)) + + summary_path = SummaryPathParser.get_latest_lineage_summaries(summary_base_dir) + if not summary_path: + log.error('There is no summary log file under summary_base_dir.') + raise LineageFileNotFoundError( + 'There is no summary log file under summary_base_dir.' + ) + + try: + result = Querier(summary_path).filter_summary_lineage( + condition=search_condition + ) + except LineageSummaryParseException: + result = {'object': [], 'count': 0} + except (LineageQuerierParamException, LineageParamTypeError) as error: + log.error(str(error)) + log.exception(error) + raise LineageQuerySummaryDataError("Filter summary lineage failed.") + + return result + + +def _convert_relative_path_to_abspath(summary_base_dir, search_condition): + """ + Convert relative path to absolute path. + + Args: + summary_base_dir (str): The summary base directory. + search_condition (dict): The search condition. + + Returns: + dict, the updated search_condition. + + Raises: + LineageParamValueError: If the value of input_name is invalid. + """ + if ("summary_dir" not in search_condition) or (not search_condition.get("summary_dir")): + return search_condition + + summary_dir_condition = search_condition.get("summary_dir") + if not set(summary_dir_condition.keys()).issubset(['in', 'eq']): + raise LineageParamValueError("Invalid operation of summary dir.") + + if 'in' in summary_dir_condition: + summary_paths = [] + for summary_dir in summary_dir_condition.get('in'): + if summary_dir.startswith('./'): + abs_dir = os.path.join( + summary_base_dir, summary_dir[2:] + ) + abs_dir = validate_path(abs_dir) + else: + abs_dir = validate_path(summary_dir) + summary_paths.append(abs_dir) + search_condition.get('summary_dir')['in'] = summary_paths + + if 'eq' in summary_dir_condition: + summary_dir = summary_dir_condition.get('eq') + if summary_dir.startswith('./'): + abs_dir = os.path.join( + summary_base_dir, summary_dir[2:] + ) + abs_dir = validate_path(abs_dir) + else: + abs_dir = validate_path(summary_dir) + search_condition.get('summary_dir')['eq'] = abs_dir + + return search_condition diff --git a/mindinsight/lineagemgr/collection/__init__.py b/mindinsight/lineagemgr/collection/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/collection/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/collection/model/__init__.py b/mindinsight/lineagemgr/collection/model/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/collection/model/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/collection/model/base.py b/mindinsight/lineagemgr/collection/model/base.py new file mode 100644 index 00000000..eec274f1 --- /dev/null +++ b/mindinsight/lineagemgr/collection/model/base.py @@ -0,0 +1,37 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Metadata of lineage collection.""" + + +class Metadata: + """Initialize parameters used in model lineage management.""" + train_dataset_path = 'train_dataset_path' + valid_dataset_path = 'valid_dataset_path' + train_network = 'train_network' + loss_function = 'loss_function' + loss = 'loss' + optimizer = 'optimizer' + learning_rate = 'learning_rate' + epoch = 'epoch' + step_num = 'step_num' + parallel_mode = 'parallel_mode' + device_num = 'device_num' + batch_size = 'batch_size' + model_path = 'model_path' + model_ckpt = 'model_ckpt' + model_size = 'model_size' + metrics = 'metrics' + train_dataset_size = 'train_dataset_size' + valid_dataset_size = 'valid_dataset_size' diff --git a/mindinsight/lineagemgr/collection/model/model_lineage.py b/mindinsight/lineagemgr/collection/model/model_lineage.py new file mode 100644 index 00000000..af8ae35c --- /dev/null +++ b/mindinsight/lineagemgr/collection/model/model_lineage.py @@ -0,0 +1,621 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This module is used to collect lineage information of model training.""" +import json +import os + +import numpy as np + +from mindinsight.lineagemgr.summary.summary_record import LineageSummary +from mindinsight.utils.exceptions import \ + MindInsightException +from mindinsight.lineagemgr.common.validator.validate import validate_train_run_context, \ + validate_eval_run_context, validate_file_path, validate_network, \ + validate_int_params, validate_summary_record, validate_raise_exception +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ + LineageGetModelFileError, LineageLogError +from mindinsight.lineagemgr.common.log import logger as log +from mindinsight.lineagemgr.common.utils import try_except +from mindinsight.lineagemgr.common.validator.model_parameter import RunContextArgs, \ + EvalParameter +from mindinsight.lineagemgr.collection.model.base import Metadata + +try: + from mindspore.common.tensor import Tensor + from mindspore.train.callback import Callback, RunContext, ModelCheckpoint, SummaryStep + from mindspore.nn import Cell, Optimizer, WithLossCell, TrainOneStepWithLossScaleCell + from mindspore.nn.loss.loss import _Loss + from mindspore.dataset.engine import Dataset, MindDataset + import mindspore.dataset as ds +except (ImportError, ModuleNotFoundError): + log.warning('MindSpore Not Found!') + + +class TrainLineage(Callback): + """ + Collect lineage of a training job. + + Args: + summary_record (SummaryRecord): SummaryRecord is used to record + the summary value, and summary_record is an instance of SummaryRecord, + see mindspore.train.summary.SummaryRecord. + raise_exception (bool): Whether to raise exception when error occurs in + TrainLineage. If True, raise exception. If False, catch exception + and continue. Default: False. + + Raises: + MindInsightException: If validating parameter fails. + LineageLogError: If recording lineage information fails. + + Examples: + >>> from mindinsight.lineagemgr import TrainLineage + >>> from mindspore.train.callback import ModelCheckpoint, SummaryStep + >>> from mindspore.train.summary import SummaryRecord + >>> model = Model(train_network) + >>> model_ckpt = ModelCheckpoint(directory='/dir/to/save/model/') + >>> summary_writer = SummaryRecord(log_dir='./') + >>> summary_callback = SummaryStep(summary_writer, flush_step=2) + >>> lineagemgr = TrainLineage(summary_record=summary_writer) + >>> model.train(epoch_num, dataset, callbacks=[model_ckpt, summary_callback, lineagemgr]) + """ + def __init__(self, summary_record, raise_exception=False): + super(TrainLineage, self).__init__() + try: + validate_raise_exception(raise_exception) + self.raise_exception = raise_exception + + validate_summary_record(summary_record) + self.summary_record = summary_record + + summary_log_path = summary_record.full_file_name + validate_file_path(summary_log_path) + self.lineage_log_path = summary_log_path + '_lineage' + + self.initial_learning_rate = None + except MindInsightException as err: + log.error(err) + if raise_exception: + raise + + @try_except(log) + def begin(self, run_context): + """ + Initialize the training progress when the training job begins. + + Args: + run_context (RunContext): It contains all lineage information, + see mindspore.train.callback.RunContext. + + Raises: + MindInsightException: If validating parameter fails. + """ + log.info('Initialize training lineage collection...') + + if not isinstance(run_context, RunContext): + error_msg = f'Invalid TrainLineage run_context.' + log.error(error_msg) + raise LineageParamRunContextError(error_msg) + + run_context_args = run_context.original_args() + if not self.initial_learning_rate: + optimizer = run_context_args.get('optimizer') + if optimizer and not isinstance(optimizer, Optimizer): + log.error("The parameter optimizer is invalid. It should be an instance of " + "mindspore.nn.optim.optimizer.Optimizer.") + raise MindInsightException(error=LineageErrors.PARAM_OPTIMIZER_ERROR, + message=LineageErrorMsg.PARAM_OPTIMIZER_ERROR.value) + if optimizer: + log.info('Obtaining initial learning rate...') + self.initial_learning_rate = AnalyzeObject.analyze_optimizer(optimizer) + log.debug('initial_learning_rate: %s', self.initial_learning_rate) + else: + network = run_context_args.get('train_network') + validate_network(network) + optimizer = AnalyzeObject.get_optimizer_by_network(network) + self.initial_learning_rate = AnalyzeObject.analyze_optimizer(optimizer) + log.debug('initial_learning_rate: %s', self.initial_learning_rate) + + # get train dataset graph + train_dataset = run_context_args.get('train_dataset') + dataset_graph_dict = ds.serialize(train_dataset) + dataset_graph_json_str = json.dumps(dataset_graph_dict, indent=2) + dataset_graph_dict = json.loads(dataset_graph_json_str) + log.info('Logging dataset graph...') + try: + lineage_summary = LineageSummary(self.lineage_log_path) + lineage_summary.record_dataset_graph(dataset_graph=dataset_graph_dict) + except Exception as error: + error_msg = f'Dataset graph log error in TrainLineage begin: {error}' + log.error(error_msg) + raise LineageLogError(error_msg) + log.info('Dataset graph logged successfully.') + + @try_except(log) + def end(self, run_context): + """ + Collect lineage information when the training job ends. + + Args: + run_context (RunContext): It contains all lineage information, + see mindspore.train.callback.RunContext. + + Raises: + LineageLogError: If recording lineage information fails. + """ + log.info('Start to collect training lineage...') + if not isinstance(run_context, RunContext): + error_msg = f'Invalid TrainLineage run_context.' + log.error(error_msg) + raise LineageParamRunContextError(error_msg) + + run_context_args = run_context.original_args() + validate_train_run_context(RunContextArgs, run_context_args) + + train_lineage = dict() + train_lineage = AnalyzeObject.get_network_args( + run_context_args, train_lineage + ) + + train_dataset = run_context_args.get('train_dataset') + callbacks = run_context_args.get('list_callback') + list_callback = getattr(callbacks, '_callbacks', []) + + log.info('Obtaining model files...') + ckpt_file_path, _ = AnalyzeObject.get_file_path(list_callback) + + train_lineage[Metadata.learning_rate] = self.initial_learning_rate + train_lineage[Metadata.epoch] = run_context_args.get('epoch_num') + train_lineage[Metadata.step_num] = run_context_args.get('cur_step_num') + train_lineage[Metadata.parallel_mode] = run_context_args.get('parallel_mode') + train_lineage[Metadata.device_num] = run_context_args.get('device_number') + train_lineage[Metadata.batch_size] = run_context_args.get('batch_num') + model_path_dict = { + 'ckpt': ckpt_file_path + } + train_lineage[Metadata.model_path] = json.dumps(model_path_dict) + + log.info('Calculating model size...') + train_lineage[Metadata.model_size] = AnalyzeObject.get_model_size( + ckpt_file_path + ) + log.debug('model_size: %s', train_lineage[Metadata.model_size]) + + log.info('Analyzing dataset object...') + train_lineage = AnalyzeObject.analyze_dataset(train_dataset, train_lineage, 'train') + + log.info('Logging lineage information...') + try: + lineage_summary = LineageSummary(self.lineage_log_path) + lineage_summary.record_train_lineage(train_lineage) + except IOError as error: + error_msg = f'End error in TrainLineage: {error}' + log.error(error_msg) + raise LineageLogError(error_msg) + except Exception as error: + error_msg = f'End error in TrainLineage: {error}' + log.error(error_msg) + log.error('Fail to log the lineage of the training job.') + raise LineageLogError(error_msg) + log.info('The lineage of the training job has logged successfully.') + + +class EvalLineage(Callback): + """ + Collect lineage of an evaluation job. + + Args: + summary_record (SummaryRecord): SummaryRecord is used to record + the summary value, and summary_record is an instance of SummaryRecord, + see mindspore.train.summary.SummaryRecord. + raise_exception (bool): Whether to raise exception when error occurs in + EvalLineage. If True, raise exception. If False, catch exception + and continue. Default: False. + + Raises: + MindInsightException: If validating parameter fails. + LineageLogError: If recording lineage information fails. + + Examples: + >>> from mindinsight.lineagemgr import EvalLineage + >>> from mindspore.train.callback import ModelCheckpoint, SummaryStep + >>> from mindspore.train.summary import SummaryRecord + >>> model = Model(train_network) + >>> model_ckpt = ModelCheckpoint(directory='/dir/to/save/model/') + >>> summary_writer = SummaryRecord(log_dir='./') + >>> summary_callback = SummaryStep(summary_writer, flush_step=2) + >>> lineagemgr = EvalLineage(summary_record=summary_writer) + >>> model.eval(epoch_num, dataset, callbacks=[model_ckpt, summary_callback, lineagemgr]) + """ + def __init__(self, summary_record, raise_exception=False): + super(EvalLineage, self).__init__() + try: + validate_raise_exception(raise_exception) + self.raise_exception = raise_exception + + validate_summary_record(summary_record) + self.summary_record = summary_record + + summary_log_path = summary_record.full_file_name + validate_file_path(summary_log_path) + self.lineage_log_path = summary_log_path + '_lineage' + except MindInsightException as err: + log.error(err) + if raise_exception: + raise + + @try_except(log) + def end(self, run_context): + """ + Collect lineage information when the training job ends. + + Args: + run_context (RunContext): It contains all lineage information, + see mindspore.train.callback.RunContext. + + Raises: + MindInsightException: If validating parameter fails. + LineageLogError: If recording lineage information fails. + """ + if not isinstance(run_context, RunContext): + error_msg = f'Invalid EvalLineage run_context.' + log.error(error_msg) + raise LineageParamRunContextError(error_msg) + + run_context_args = run_context.original_args() + validate_eval_run_context(EvalParameter, run_context_args) + + valid_dataset = run_context_args.get('valid_dataset') + + eval_lineage = dict() + metrics = run_context_args.get('metrics') + eval_lineage[Metadata.metrics] = json.dumps(metrics) + eval_lineage[Metadata.step_num] = run_context_args.get('cur_step_num') + + log.info('Analyzing dataset object...') + eval_lineage = AnalyzeObject.analyze_dataset(valid_dataset, eval_lineage, 'valid') + + log.info('Logging evaluation job lineage...') + try: + lineage_summary = LineageSummary(self.lineage_log_path) + lineage_summary.record_evaluation_lineage(eval_lineage) + except IOError as error: + error_msg = f'End error in EvalLineage: {error}' + log.error(error_msg) + log.error('Fail to log the lineage of the evaluation job.') + raise LineageLogError(error_msg) + except Exception as error: + error_msg = f'End error in EvalLineage: {error}' + log.error(error_msg) + log.error('Fail to log the lineage of the evaluation job.') + raise LineageLogError(error_msg) + log.info('The lineage of the evaluation job has logged successfully.') + + +class AnalyzeObject: + """Analyze class object in MindSpore.""" + + @staticmethod + def get_optimizer_by_network(network): + """ + Get optimizer by analyzing network. + + Args: + network (Cell): See mindspore.nn.Cell. + + Returns: + Optimizer, an Optimizer object. + """ + optimizer = None + net_args = vars(network) if network else {} + net_cell = net_args.get('_cells') if net_args else {} + for _, value in net_cell.items(): + if isinstance(value, Optimizer): + optimizer = value + break + return optimizer + + @staticmethod + def get_loss_fn_by_network(network): + """ + Get loss function by analyzing network. + + Args: + network (Cell): See mindspore.nn.Cell. + + Returns: + Loss_fn, a Cell object. + """ + loss_fn = None + inner_cell_list = [] + net_args = vars(network) if network else {} + net_cell = net_args.get('_cells') if net_args else {} + for _, value in net_cell.items(): + if isinstance(value, Cell) and \ + not isinstance(value, Optimizer): + inner_cell_list.append(value) + + while inner_cell_list: + inner_net_args = vars(inner_cell_list[0]) + inner_net_cell = inner_net_args.get('_cells') + + for value in inner_net_cell.values(): + if isinstance(value, _Loss): + loss_fn = value + break + if isinstance(value, Cell): + inner_cell_list.append(value) + if loss_fn: + break + + inner_cell_list.pop(0) + + return loss_fn + + @staticmethod + def get_backbone_network(network): + """ + Get the name of backbone network. + + Args: + network (Cell): The train network. + + Returns: + str, the name of the backbone network. + """ + with_loss_cell = False + backbone = None + net_args = vars(network) if network else {} + net_cell = net_args.get('_cells') if net_args else {} + + for _, value in net_cell.items(): + if isinstance(value, WithLossCell): + backbone = getattr(value, '_backbone') + with_loss_cell = True + break + + if with_loss_cell: + backbone_name = type(backbone).__name__ \ + if backbone else None + elif isinstance(network, TrainOneStepWithLossScaleCell): + backbone = getattr(network, 'network') + backbone_name = type(backbone).__name__ \ + if backbone else None + else: + backbone_name = type(network).__name__ \ + if network else None + return backbone_name + + @staticmethod + def analyze_optimizer(optimizer): + """ + Analyze Optimizer, a Cell object of MindSpore. + + In this way, we can obtain the following attributes: + learning_rate (float), + weight_decay (float), + momentum (float), + weights (float). + + Args: + optimizer (Optimizer): See mindspore.nn.optim.Optimizer. + + Returns: + float, the learning rate that the optimizer adopted. + """ + learning_rate = None + if isinstance(optimizer, Optimizer): + learning_rate = getattr(optimizer, 'learning_rate', None) + + if learning_rate: + learning_rate = learning_rate.default_input + + # Get the real learning rate value + if isinstance(learning_rate, Tensor): + learning_rate = learning_rate.asnumpy() + if learning_rate.ndim == 0: + learning_rate = np.atleast_1d(learning_rate) + learning_rate = list(learning_rate) + elif isinstance(learning_rate, float): + learning_rate = [learning_rate] + + return learning_rate[0] if learning_rate else None + + @staticmethod + def analyze_dataset(dataset, lineage_dict, dataset_type): + """ + Analyze Dataset, a Dataset object of MindSpore. + + In this way, we can obtain the following attributes: + dataset_path (str), + train_dataset_size (int), + valid_dataset_size (int), + batch_size (int) + + Args: + dataset (Dataset): See mindspore.dataengine.datasets.Dataset. + lineage_dict (dict): A dict contains lineage metadata. + dataset_type (str): Dataset type, train or valid. + + Returns: + dict, the lineage metadata. + """ + dataset_batch_size = dataset.get_dataset_size() + if dataset_batch_size is not None: + validate_int_params(dataset_batch_size, 'dataset_batch_size') + log.debug('dataset_batch_size: %d', dataset_batch_size) + dataset_path = AnalyzeObject.get_dataset_path_wrapped(dataset) + if dataset_path: + dataset_path = '/'.join(dataset_path.split('/')[:-1]) + + step_num = lineage_dict.get('step_num') + validate_int_params(step_num, 'step_num') + log.debug('step_num: %d', step_num) + + if dataset_type == 'train': + lineage_dict[Metadata.train_dataset_path] = dataset_path + epoch = lineage_dict.get('epoch') + train_dataset_size = dataset_batch_size * (step_num / epoch) + lineage_dict[Metadata.train_dataset_size] = int(train_dataset_size) + elif dataset_type == 'valid': + lineage_dict[Metadata.valid_dataset_path] = dataset_path + lineage_dict[Metadata.valid_dataset_size] = dataset_batch_size * step_num + + return lineage_dict + + def get_dataset_path(self, output_dataset): + """ + Get dataset path of MindDataset object. + + Args: + output_dataset (Union[MindDataset, Dataset]): See + mindspore.dataengine.datasets.Dataset. + + Returns: + str, dataset path. + """ + if isinstance(output_dataset, MindDataset): + return output_dataset.dataset_file + return self.get_dataset_path(output_dataset.input[0]) + + @staticmethod + def get_dataset_path_wrapped(dataset): + """ + A wrapper for obtaining dataset path. + + Args: + dataset (Union[MindDataset, Dataset]): See + mindspore.dataengine.datasets.Dataset. + + Returns: + str, dataset path. + """ + dataset_path = None + if isinstance(dataset, Dataset): + try: + dataset_path = AnalyzeObject().get_dataset_path(dataset) + except IndexError: + dataset_path = None + validate_file_path(dataset_path, allow_empty=True) + return dataset_path + + @staticmethod + def get_file_path(list_callback): + """ + Get ckpt_file_name and summary_log_path from MindSpore callback list. + + Args: + list_callback (list[Callback]): The MindSpore training Callback list. + + Returns: + tuple, contains ckpt_file_name and summary_log_path. + """ + ckpt_file_path = None + summary_log_path = None + for callback in list_callback: + if isinstance(callback, ModelCheckpoint): + ckpt_file_path = callback.latest_ckpt_file_name + if isinstance(callback, SummaryStep): + summary_log_path = callback.summary_file_name + + if ckpt_file_path: + validate_file_path(ckpt_file_path) + ckpt_file_path = os.path.realpath(ckpt_file_path) + + if summary_log_path: + validate_file_path(summary_log_path) + summary_log_path = os.path.realpath(summary_log_path) + + return ckpt_file_path, summary_log_path + + @staticmethod + def get_file_size(file_path): + """ + Get the file size. + + Args: + file_path (str): The file path. + + Returns: + int, the file size. + """ + try: + return os.path.getsize(file_path) + except (OSError, IOError) as error: + error_msg = f"Error when get model file size: {error}" + log.error(error_msg) + raise LineageGetModelFileError(error_msg) + + @staticmethod + def get_model_size(ckpt_file_path): + """ + Get model the total size of the model file and the checkpoint file. + + Args: + ckpt_file_path (str): The checkpoint file path. + + Returns: + int, the total file size. + """ + if ckpt_file_path: + ckpt_file_path = os.path.realpath(ckpt_file_path) + ckpt_file_size = AnalyzeObject.get_file_size(ckpt_file_path) + else: + ckpt_file_size = 0 + + return ckpt_file_size + + @staticmethod + def get_network_args(run_context_args, train_lineage): + """ + Get the parameters related to the network, + such as optimizer, loss function. + + Args: + run_context_args (dict): It contains all information of the training job. + train_lineage (dict): A dict contains lineage metadata. + + Returns: + dict, the lineage metadata. + """ + network = run_context_args.get('train_network') + validate_network(network) + optimizer = run_context_args.get('optimizer') + if not optimizer: + optimizer = AnalyzeObject.get_optimizer_by_network(network) + loss_fn = run_context_args.get('loss_fn') + if not loss_fn: + loss_fn = AnalyzeObject.get_loss_fn_by_network(network) + loss = None + else: + loss = run_context_args.get('net_outputs') + + if loss: + log.info('Calculating loss...') + loss_numpy = loss.asnumpy() + loss = float(np.atleast_1d(loss_numpy)[0]) + log.debug('loss: %s', loss) + train_lineage[Metadata.loss] = loss + else: + train_lineage[Metadata.loss] = None + + # Analyze classname of optimizer, loss function and training network. + train_lineage[Metadata.optimizer] = type(optimizer).__name__ \ + if optimizer else None + train_lineage[Metadata.train_network] = AnalyzeObject.get_backbone_network(network) + train_lineage[Metadata.loss_function] = type(loss_fn).__name__ \ + if loss_fn else None + + return train_lineage diff --git a/mindinsight/lineagemgr/common/__init__.py b/mindinsight/lineagemgr/common/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/common/exceptions/__init__.py b/mindinsight/lineagemgr/common/exceptions/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/common/exceptions/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/common/exceptions/error_code.py b/mindinsight/lineagemgr/common/exceptions/error_code.py new file mode 100644 index 00000000..fe81a6e8 --- /dev/null +++ b/mindinsight/lineagemgr/common/exceptions/error_code.py @@ -0,0 +1,207 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lineage error code and messages.""" +from enum import Enum, unique +from mindinsight.utils.constant import LineageMgrErrors as LineageErrorCodes + + +_PARAM_ERROR_MASK = 0b00001 << 7 +_MINDSPORE_COLLECTOR_ERROR = 0b00011 << 7 +_MODEL_LINEAGE_API_ERROR_MASK = 0b00100 << 7 +_DATASET_COLLECTOR_ERROR_MASK = 0b00101 << 7 +_DATASET_LINEAGE_ERROR_MASK = 0b00110 << 7 +_SUMMARY_ANALYZE_ERROR_MASK = 0b00111 << 7 +_QUERIER_ERROR_MASK = 0b01000 << 7 + + +@unique +class LineageErrors(LineageErrorCodes): + """Lineage error codes.""" + PARAM_TYPE_ERROR = 0 | _PARAM_ERROR_MASK + PARAM_VALUE_ERROR = 1 | _PARAM_ERROR_MASK + PARAM_MISSING_ERROR = 2 | _PARAM_ERROR_MASK + PARAM_SUMMARY_RECORD_ERROR = 3 | _PARAM_ERROR_MASK + PARAM_RAISE_EXCEPTION_ERROR = 4 | _PARAM_ERROR_MASK + + # MindSpore Collector error codes. + PARAM_RUN_CONTEXT_ERROR = 0 | _MINDSPORE_COLLECTOR_ERROR + PARAM_OPTIMIZER_ERROR = 1 | _MINDSPORE_COLLECTOR_ERROR + PARAM_LOSS_FN_ERROR = 2 | _MINDSPORE_COLLECTOR_ERROR + PARAM_TRAIN_NETWORK_ERROR = 3 | _MINDSPORE_COLLECTOR_ERROR + PARAM_DATASET_ERROR = 4 | _MINDSPORE_COLLECTOR_ERROR + PARAM_EPOCH_NUM_ERROR = 5 | _MINDSPORE_COLLECTOR_ERROR + PARAM_BATCH_NUM_ERROR = 6 | _MINDSPORE_COLLECTOR_ERROR + PARAM_TRAIN_PARALLEL_ERROR = 7 | _MINDSPORE_COLLECTOR_ERROR + PARAM_DEVICE_NUMBER_ERROR = 8 | _MINDSPORE_COLLECTOR_ERROR + PARAM_FILE_PATH_ERROR = 9 | _MINDSPORE_COLLECTOR_ERROR + PARAM_DATASET_SIZE_ERROR = 10 | _MINDSPORE_COLLECTOR_ERROR + PARAM_LEARNING_RATE_ERROR = 11 | _MINDSPORE_COLLECTOR_ERROR + PARAM_EVAL_METRICS_ERROR = 12 | _MINDSPORE_COLLECTOR_ERROR + PARAM_BATCH_SIZE_ERROR = 13 | _MINDSPORE_COLLECTOR_ERROR + PARAM_NET_OUTPUTS_ERROR = 14 | _MINDSPORE_COLLECTOR_ERROR + PARAM_CALLBACK_LIST_ERROR = 15 | _MINDSPORE_COLLECTOR_ERROR + LINEAGE_GET_MODEL_FILE_ERROR = 16 | _MINDSPORE_COLLECTOR_ERROR + LOG_LINEAGE_INFO_ERROR = 17 | _MINDSPORE_COLLECTOR_ERROR + PARAM_STEP_NUM_ERROR = 18 | _MINDSPORE_COLLECTOR_ERROR + + # Model lineage error codes. + LINEAGE_PARAM_OPERATION_ERROR = 0 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_METRIC_ERROR = 1 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_LOSS_FUNCTION_ERROR = 4 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_TRAIN_DATASET_PATH_ERROR = 5 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_TRAIN_DATASET_COUNT_ERROR = 6 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_TEST_DATASET_PATH_ERROR = 7 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_TEST_DATASET_COUNT_ERROR = 8 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_NETWORK_ERROR = 9 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_OPTIMIZER_ERROR = 10 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_LEARNING_RATE_ERROR = 11 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_EPOCH_ERROR = 12 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_BATCH_SIZE_ERROR = 13 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_NOT_SUPPORT_ERROR = 14 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_LOSS_ERROR = 15 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_MODEL_SIZE_ERROR = 16 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_SUMMARY_DIR_ERROR = 17 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_SORTED_NAME_ERROR = 18 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_SORTED_TYPE_ERROR = 19 | _MODEL_LINEAGE_API_ERROR_MASK + + LINEAGE_DIR_NOT_EXIST_ERROR = 20 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_SUMMARY_DATA_ERROR = 21 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_FILE_NOT_FOUND_ERROR = 22 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_SUMMARY_PATH_ERROR = 23 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_SEARCH_CONDITION_PARAM_ERROR = 24 | _MODEL_LINEAGE_API_ERROR_MASK + LINEAGE_PARAM_LINEAGE_TYPE_ERROR = 25 | _MODEL_LINEAGE_API_ERROR_MASK + + + SUMMARY_ANALYZE_ERROR = 0 | _SUMMARY_ANALYZE_ERROR_MASK + SUMMARY_VERIFICATION_ERROR = 1 | _SUMMARY_ANALYZE_ERROR_MASK + + # Querier error codes. + EVENT_NOT_EXIST_ERROR = 0 | _QUERIER_ERROR_MASK + QUERIER_PARAM_ERROR = 1 | _QUERIER_ERROR_MASK + SUMMARY_PARSE_FAIL_ERROR = 2 | _QUERIER_ERROR_MASK + EVENT_FIELD_NOT_EXIST_ERROR = 4 | _QUERIER_ERROR_MASK + + +@unique +class LineageErrorMsg(Enum): + """Lineage error messages.""" + PARAM_TYPE_ERROR = "TypeError. {}" + PARAM_VALUE_ERROR = "ValueError. {}" + PARAM_MISSING_ERROR = "MissingError. {}" + PARAM_LIMIT_ERROR = "Invalid input limit. 0 < limit <= 100" + PARAM_OFFSET_ERROR = "Invalid input offset. 0 <= offset <= 100000" + PARAM_SUMMARY_RECORD_ERROR = "Invalid value for summary_record. It should be an instance of " \ + "mindspore.train.summary.SummaryRecord" + PARAM_RAISE_EXCEPTION_ERROR = "Invalid value for raise_exception. It should be True or False." + # Lineage error messages. + LINEAGE_PARAM_SUMMARY_PATH_ERROR = "The parameter summary path error: {}" + LINEAGE_SUMMARY_DATA_ERROR = "Query summary data error: {}" + LINEAGE_FILE_NOT_FOUND_ERROR = "File not found error: {}" + LINEAGE_DIR_NOT_EXIST_ERROR = "Dir not exist error: {}" + LINEAGE_SEARCH_CONDITION_PARAM_ERROR = "Search_condition param error: {}" + + # MindSpore Collector error messages. + PARAM_RUN_CONTEXT_ERROR = "The parameter run_context is invalid. It should be an instance of " \ + "mindspore.train.callback.RunContext. {}" + + PARAM_OPTIMIZER_ERROR = "The parameter optimizer is invalid. It should be an instance of " \ + "mindspore.nn.optim.optimizer.Optimizer." + + PARAM_LOSS_FN_ERROR = "The parameter loss_fn is invalid. It should be a Function." + + PARAM_NET_OUTPUTS_ERROR = "The parameter net_outputs is invalid. It should be a Tensor." + + PARAM_TRAIN_NETWORK_ERROR = "The parameter train_network is invalid. It should be an instance of " \ + "mindspore.nn.cell.Cell." + + PARAM_EPOCH_NUM_ERROR = "The parameter epoch is invalid. It should be a positive integer." + + PARAM_STEP_NUM_ERROR = "The parameter step_num is invalid. It should be a positive integer." + + PARAM_BATCH_NUM_ERROR = "The parameter batch_num is invalid. It should be a non-negative integer." + + PARAM_TRAIN_PARALLEL_ERROR = "The parameter parallel_mode is invalid. It should be an integer" \ + "between 0 and 4." + + PARAM_DEVICE_NUMBER_ERROR = "The parameter device_number is invalid. It should be a positive integer." + + PARAM_LEARNING_RATE_ERROR = "The parameter learning_rate is invalid. It should be a float number or " \ + "an instance of mindspore.common.tensor.Tensor." + + PARAM_EVAL_METRICS_ERROR = "The parameter metrics is invalid. It should be a dictionary." + + PARAM_BATCH_SIZE_ERROR = "The parameter batch_size is invalid. It should be a non-negative integer." + + PARAM_CALLBACK_LIST_ERROR = "The parameter list_callback is invalid. It should be an instance of " \ + "mindspore.train.callback._ListCallback." + + LINEAGE_GET_MODEL_FILE_ERROR = "Error when get model file size. {}" + + LINEAGE_METRIC_ERROR = "The parameter {} is invalid. " \ + "It should be a dict and the value should be a float or a integer" + + LINEAGE_COMPARE_OPERATION_ERROR = "The schema error and compare operation should be" \ + " 'eq', 'lt', 'gt', 'ge', 'le', 'in'." + + LINEAGE_PARAM_SUMMARY_DIR_ERROR = "The parameter summary_dir is invalid. It should be a dict and the value " \ + "should be a string" + + LINEAGE_TRAIN_DATASET_PATH_ERROR = "The parameter train_dataset_path is invalid." \ + " It should be a dict and the value should be a string" + + LINEAGE_TRAIN_DATASET_COUNT_ERROR = "The parameter train_dataset_count is invalid. It should be a dict " \ + "and the value should be a integer between 0 and pow(2, 63) -1" + + LINEAGE_TEST_DATASET_PATH_ERROR = "The parameter test_dataset_path is invalid. " \ + "It should be a dict and the value should be a string" + + LINEAGE_TEST_DATASET_COUNT_ERROR = "The parameter test_dataset_count is invalid. It should be a dict " \ + "and the value should be a integer between 0 and pow(2, 63) -1" + + LINEAGE_NETWORK_ERROR = "The parameter network is invalid. It should be a dict and the value should be a string" + + LINEAGE_OPTIMIZER_ERROR = "The parameter optimizer is invalid. It should be a dict and the value should be a string" + + LINEAGE_LOSS_FUNCTION_ERROR = "The parameter loss_function is invalid. " \ + "It should be a dict and the value should be a string" + + LINEAGE_LOSS_ERROR = "The parameter loss is invalid. " \ + "It should be a float." + + LINEAGE_MODEL_SIZE_ERROR = "The parameter model_size is invalid. " \ + "It should be an integer between 0 and pow(2, 63) -1." + + LINEAGE_LEARNING_RATE_ERROR = "The parameter learning_rate is invalid. " \ + "It should be a dict and the value should be a float or a integer" + + LINEAGE_PARAM_SORTED_NAME_ERROR = "The parameter sorted_name is invalid. " \ + "It should be a string." + + LINEAGE_PARAM_SORTED_TYPE_ERROR = "The parameter sorted_type is invalid. " \ + "It should be a string." + + LINEAGE_PARAM_LINEAGE_TYPE_ERROR = "The parameter lineage_type is invalid. " \ + "It should be None, 'dataset' or 'model'." + + SUMMARY_ANALYZE_ERROR = "Failed to analyze summary log. {}" + SUMMARY_VERIFICATION_ERROR = "Verification failed in summary analysis. {}" + + # Querier error codes. + EVENT_NOT_EXIST_ERROR = "Train and evaluation event not exist in summary log." + QUERIER_PARAM_ERROR = "Querier param <{}> invalid. {}" + SUMMARY_PARSE_FAIL_ERROR = "All summary logs parsing failed." + EVENT_FIELD_NOT_EXIST_ERROR = 'Event field <{}> not exist.' + + LOG_LINEAGE_INFO_ERROR = "Fail to write lineage information into log file. {}" diff --git a/mindinsight/lineagemgr/common/exceptions/exceptions.py b/mindinsight/lineagemgr/common/exceptions/exceptions.py new file mode 100644 index 00000000..e9053230 --- /dev/null +++ b/mindinsight/lineagemgr/common/exceptions/exceptions.py @@ -0,0 +1,191 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Definition of error code and relative messages in lineage module.""" +from mindinsight.utils.exceptions import MindInsightException +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg + + +class LineageParamTypeError(MindInsightException): + """The parameter type error in lineage module.""" + + def __init__(self, msg): + super(LineageParamTypeError, self).__init__( + error=LineageErrors.PARAM_TYPE_ERROR, + message=LineageErrorMsg.PARAM_TYPE_ERROR.value.format(msg) + ) + + +class LineageParamValueError(MindInsightException): + """The parameter value error in lineage module.""" + + def __init__(self, msg): + super(LineageParamValueError, self).__init__( + error=LineageErrors.PARAM_VALUE_ERROR, + message=LineageErrorMsg.PARAM_VALUE_ERROR.value.format(msg) + ) + + +class LineageParamMissingError(MindInsightException): + """The parameter missing error in lineage module.""" + + def __init__(self, msg): + super(LineageParamMissingError, self).__init__( + error=LineageErrors.PARAM_MISSING_ERROR, + message=LineageErrorMsg.PARAM_MISSING_ERROR.value.format(msg) + ) + + +class LineageParamRunContextError(MindInsightException): + """The input parameter run_context error in lineage module.""" + + def __init__(self, msg): + super(LineageParamRunContextError, self).__init__( + error=LineageErrors.PARAM_RUN_CONTEXT_ERROR, + message=LineageErrorMsg.PARAM_RUN_CONTEXT_ERROR.value.format(msg) + ) + + +class LineageGetModelFileError(MindInsightException): + """The get model file error in lineage module.""" + + def __init__(self, msg): + super(LineageGetModelFileError, self).__init__( + error=LineageErrors.LINEAGE_GET_MODEL_FILE_ERROR, + message=LineageErrorMsg.LINEAGE_GET_MODEL_FILE_ERROR.value.format(msg) + ) + + +class LineageSearchModelParamError(MindInsightException): + """The lineage search model param error.""" + def __init__(self, msg): + super(LineageSearchModelParamError, self).__init__( + error=LineageErrors.LINEAGE_PARAM_NOT_SUPPORT_ERROR, + message=LineageErrorMsg.LINEAGE_PARAM_NOT_SUPPORT_ERROR.value.format(msg) + ) + + +class LineageSummaryAnalyzeException(MindInsightException): + """The summary analyze error in lineage module.""" + + def __init__(self, msg=None): + if msg is None: + msg = '' + super(LineageSummaryAnalyzeException, self).__init__( + error=LineageErrors.SUMMARY_ANALYZE_ERROR, + message=LineageErrorMsg.SUMMARY_ANALYZE_ERROR.value.format(msg) + ) + + +class LineageVerificationException(MindInsightException): + """The summary verification error in lineage module.""" + def __init__(self, msg): + super(LineageVerificationException, self).__init__( + error=LineageErrors.SUMMARY_VERIFICATION_ERROR, + message=LineageErrorMsg.SUMMARY_VERIFICATION_ERROR.value.format(msg) + ) + + +class LineageLogError(MindInsightException): + """The lineage collector error.""" + def __init__(self, msg): + super(LineageLogError, self).__init__( + error=LineageErrors.LOG_LINEAGE_INFO_ERROR, + message=LineageErrorMsg.LOG_LINEAGE_INFO_ERROR.value.format(msg) + ) + + +class LineageEventNotExistException(MindInsightException): + """The querier error in lineage module.""" + + def __init__(self): + super(LineageEventNotExistException, self).__init__( + error=LineageErrors.EVENT_NOT_EXIST_ERROR, + message=LineageErrorMsg.EVENT_NOT_EXIST_ERROR.value + ) + + +class LineageQuerierParamException(MindInsightException): + """The querier error in lineage module.""" + + def __init__(self, *msg): + super(LineageQuerierParamException, self).__init__( + error=LineageErrors.QUERIER_PARAM_ERROR, + message=LineageErrorMsg.QUERIER_PARAM_ERROR.value.format(*msg) + ) + + +class LineageSummaryParseException(MindInsightException): + """The querier error in lineage module.""" + + def __init__(self): + super(LineageSummaryParseException, self).__init__( + error=LineageErrors.SUMMARY_PARSE_FAIL_ERROR, + message=LineageErrorMsg.SUMMARY_PARSE_FAIL_ERROR.value + ) + + +class LineageEventFieldNotExistException(MindInsightException): + """The querier error in lineage module.""" + + def __init__(self, msg): + super(LineageEventFieldNotExistException, self).__init__( + error=LineageErrors.EVENT_FIELD_NOT_EXIST_ERROR, + message=LineageErrorMsg.EVENT_FIELD_NOT_EXIST_ERROR.value.format(msg) + ) + + +class LineageParamSummaryPathError(MindInsightException): + """The lineage parameter summary path error.""" + def __init__(self, msg): + super(LineageParamSummaryPathError, self).__init__( + error=LineageErrors.LINEAGE_PARAM_SUMMARY_PATH_ERROR, + message=LineageErrorMsg.LINEAGE_PARAM_SUMMARY_PATH_ERROR.value.format(msg) + ) + + +class LineageQuerySummaryDataError(MindInsightException): + """Query summary data error in lineage module.""" + def __init__(self, msg): + super(LineageQuerySummaryDataError, self).__init__( + error=LineageErrors.LINEAGE_SUMMARY_DATA_ERROR, + message=LineageErrorMsg.LINEAGE_SUMMARY_DATA_ERROR.value.format(msg) + ) + + +class LineageFileNotFoundError(MindInsightException): + """Summary file not found in lineage module.""" + def __init__(self, msg): + super(LineageFileNotFoundError, self).__init__( + error=LineageErrors.LINEAGE_FILE_NOT_FOUND_ERROR, + message=LineageErrorMsg.LINEAGE_FILE_NOT_FOUND_ERROR.value.format(msg) + ) + + +class LineageDirNotExistError(MindInsightException): + """Directory not exist in lineage module.""" + def __init__(self, msg): + super(LineageDirNotExistError, self).__init__( + error=LineageErrors.LINEAGE_DIR_NOT_EXIST_ERROR, + message=LineageErrorMsg.LINEAGE_DIR_NOT_EXIST_ERROR.value.format(msg) + ) + + +class LineageSearchConditionParamError(MindInsightException): + """Search condition param is invalid in lineage module.""" + def __init__(self, msg): + super(LineageSearchConditionParamError, self).__init__( + error=LineageErrors.LINEAGE_SEARCH_CONDITION_PARAM_ERROR, + message=LineageErrorMsg.LINEAGE_SEARCH_CONDITION_PARAM_ERROR.value.format(msg) + ) diff --git a/mindinsight/lineagemgr/common/log.py b/mindinsight/lineagemgr/common/log.py new file mode 100644 index 00000000..05084cae --- /dev/null +++ b/mindinsight/lineagemgr/common/log.py @@ -0,0 +1,20 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Import mindinsight unified log module.""" +from mindinsight.utils.log import setup_logger + +LOG_NAME = "lineage" +LOG_MODULE = "lineage" +logger = setup_logger(sub_module=LOG_MODULE, log_name=LOG_NAME) diff --git a/mindinsight/lineagemgr/common/path_parser.py b/mindinsight/lineagemgr/common/path_parser.py new file mode 100644 index 00000000..3d96b9dd --- /dev/null +++ b/mindinsight/lineagemgr/common/path_parser.py @@ -0,0 +1,149 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file provides path resolution.""" +import os + +from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + + +class SummaryPathParser: + """ + Summary path parser. + + This class is a utility class, users can use it to parse summary dir, + parse summary log path, get the latest lineage summary log, etc. + """ + LINEAGE_SUMMARY_SUFFIX = '_lineage' + _LINEAGE_SUMMARY_SUFFIX_LEN = len(LINEAGE_SUMMARY_SUFFIX) + + @staticmethod + def get_summary_dirs(summary_base_dir): + """ + Get summary dirs according to summary base dir. + + Args: + summary_base_dir (str): Summary base dir. + + Returns: + list[str], all summary dirs in summary base dir. The summary dir is + absolute path. + """ + summary_watcher = SummaryWatcher() + relative_dirs = summary_watcher.list_summary_directories( + summary_base_dir=summary_base_dir + ) + summary_dirs = list( + map( + lambda item: os.path.realpath( + os.path.join(summary_base_dir, item.get('relative_path')) + ), + relative_dirs + ) + ) + return summary_dirs + + @staticmethod + def get_latest_lineage_summary(summary_dir): + """ + Get latest lineage summary log path according to summary dir. + + Args: + summary_dir (str): Summary dir. + + Returns: + Union[str, None], if the lineage summary log exist, return the path, + else return None. The lineage summary log path is absolute path. + """ + summary_watcher = SummaryWatcher() + summaries = summary_watcher.list_summaries(summary_base_dir=summary_dir) + latest_file_name = SummaryPathParser._get_latest_lineage_file(summaries) + return os.path.join(summary_dir, latest_file_name) \ + if latest_file_name is not None else None + + @staticmethod + def get_latest_lineage_summaries(summary_base_dir): + """ + Get all latest lineage summary logs in summary base dir. + + Args: + summary_base_dir (str): Summary base dir. + + Returns: + list[str], all latest lineage summary logs in summary base dir. The + lineage summary log is absolute path. + """ + summary_watcher = SummaryWatcher() + relative_dirs = summary_watcher.list_summary_directories( + summary_base_dir=summary_base_dir + ) + latest_summaries = [] + for item in relative_dirs: + relative_dir = item.get('relative_path') + summaries = summary_watcher.list_summaries( + summary_base_dir=summary_base_dir, + relative_path=relative_dir + ) + latest_file_name = SummaryPathParser._get_latest_lineage_file( + summaries + ) + if latest_file_name is None: + continue + latest_file = os.path.realpath( + os.path.join( + summary_base_dir, + relative_dir, + latest_file_name + ) + ) + latest_summaries.append(latest_file) + return latest_summaries + + @staticmethod + def _get_latest_lineage_file(summaries): + """ + Get latest lineage summary file. + + If there is a file with the suffix `LINEAGE_SUMMARY_SUFFIX`, check + whether there is a file with the same name that does not include the + suffix `LINEAGE_SUMMARY_SUFFIX`. When both exist, the file is considered + to be a lineage summary log. + + Args: + summaries (list[dict]): All summary logs info in summary dir. + + Returns: + str, the latest lineage summary file name. + """ + try: + latest_summary = max( + summaries, + key=lambda summary: summary.get('create_time') + ) + except ValueError: + return None + max_create_time = latest_summary.get('create_time') + summary_file_names = [] + for summary in summaries: + if summary.get('create_time') == max_create_time: + summary_file_names.append(summary.get('file_name')) + + latest_lineage_name = None + for name in summary_file_names: + if not name.endswith(SummaryPathParser.LINEAGE_SUMMARY_SUFFIX): + continue + ms_name = name[:-SummaryPathParser._LINEAGE_SUMMARY_SUFFIX_LEN] + if ms_name in summary_file_names: + latest_lineage_name = name + return latest_lineage_name diff --git a/mindinsight/lineagemgr/common/utils.py b/mindinsight/lineagemgr/common/utils.py new file mode 100644 index 00000000..0fd499b2 --- /dev/null +++ b/mindinsight/lineagemgr/common/utils.py @@ -0,0 +1,56 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lineage utils.""" +from functools import wraps + +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ + LineageGetModelFileError, LineageLogError +from mindinsight.utils.exceptions import MindInsightException + + +def enum_to_list(enum): + return [enum_ele.value for enum_ele in enum] + + +def try_except(logger): + """ + Catch or raise exceptions while collecting lineage. + + Args: + logger (logger): The logger instance which logs the warning info. + + Returns: + function, the decorator which we use to retry the decorated function. + """ + def try_except_decorate(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + try: + func(self, *args, **kwargs) + except (AttributeError, MindInsightException, + LineageParamRunContextError, LineageLogError, + LineageGetModelFileError, IOError) as err: + logger.error(err) + + try: + raise_except = self.raise_exception + except AttributeError: + raise_except = False + + if raise_except is True: + raise + + return wrapper + return try_except_decorate diff --git a/mindinsight/lineagemgr/common/validator/__init__.py b/mindinsight/lineagemgr/common/validator/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/lineagemgr/common/validator/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/common/validator/model_parameter.py b/mindinsight/lineagemgr/common/validator/model_parameter.py new file mode 100644 index 00000000..87de009f --- /dev/null +++ b/mindinsight/lineagemgr/common/validator/model_parameter.py @@ -0,0 +1,253 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Define schema of model lineage input parameters.""" +from marshmallow import Schema, fields, ValidationError, pre_load, validates +from marshmallow.validate import Range, OneOf + +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrorMsg, \ + LineageErrors +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageParamTypeError, LineageParamValueError +from mindinsight.lineagemgr.common.log import logger +from mindinsight.lineagemgr.common.utils import enum_to_list +from mindinsight.lineagemgr.querier.querier import LineageType +from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING +from mindinsight.utils.exceptions import MindInsightException + +try: + from mindspore.dataset.engine import Dataset + from mindspore.nn import Cell, Optimizer + from mindspore.common.tensor import Tensor + from mindspore.train.callback import _ListCallback +except (ImportError, ModuleNotFoundError): + logger.error('MindSpore Not Found!') + + +class RunContextArgs(Schema): + """Define the parameter schema for RunContext.""" + optimizer = fields.Function(allow_none=True) + loss_fn = fields.Function(allow_none=True) + net_outputs = fields.Function(allow_none=True) + train_network = fields.Function(allow_none=True) + train_dataset = fields.Function(allow_none=True) + epoch_num = fields.Int(allow_none=True, validate=Range(min=1)) + batch_num = fields.Int(allow_none=True, validate=Range(min=0)) + cur_step_num = fields.Int(allow_none=True, validate=Range(min=0)) + parallel_mode = fields.Str(allow_none=True) + device_number = fields.Int(allow_none=True, validate=Range(min=1)) + list_callback = fields.Function(allow_none=True) + + @pre_load + def check_optimizer(self, data, **kwargs): + optimizer = data.get("optimizer") + if optimizer and not isinstance(optimizer, Optimizer): + raise ValidationError({'optimizer': [ + "Parameter optimizer must be an instance of mindspore.nn.optim.Optimizer." + ]}) + return data + + @pre_load + def check_train_network(self, data, **kwargs): + train_network = data.get("train_network") + if train_network and not isinstance(train_network, Cell): + raise ValidationError({'train_network': [ + "Parameter train_network must be an instance of mindspore.nn.Cell."]}) + return data + + @pre_load + def check_train_dataset(self, data, **kwargs): + train_dataset = data.get("train_dataset") + if train_dataset and not isinstance(train_dataset, Dataset): + raise ValidationError({'train_dataset': [ + "Parameter train_dataset must be an instance of " + "mindspore.dataengine.datasets.Dataset"]}) + return data + + @pre_load + def check_loss(self, data, **kwargs): + net_outputs = data.get("net_outputs") + if net_outputs and not isinstance(net_outputs, Tensor): + raise ValidationError({'net_outpus': [ + "The parameter net_outputs is invalid. It should be a Tensor." + ]}) + return data + + @pre_load + def check_list_callback(self, data, **kwargs): + list_callback = data.get("list_callback") + if list_callback and not isinstance(list_callback, _ListCallback): + raise ValidationError({'list_callback': [ + "Parameter list_callback must be an instance of " + "mindspore.train.callback._ListCallback." + ]}) + return data + + +class EvalParameter(Schema): + """Define the parameter schema for Evaluation job.""" + valid_dataset = fields.Function(allow_none=True) + metrics = fields.Dict(allow_none=True) + + @pre_load + def check_valid_dataset(self, data, **kwargs): + valid_dataset = data.get("valid_dataset") + if valid_dataset and not isinstance(valid_dataset, Dataset): + raise ValidationError({'valid_dataset': [ + "Parameter valid_dataset must be an instance of " + "mindspore.dataengine.datasets.Dataset"]}) + return data + + +class SearchModelConditionParameter(Schema): + """Define the search model condition parameter schema.""" + summary_dir = fields.Dict() + loss_function = fields.Dict() + train_dataset_path = fields.Dict() + train_dataset_count = fields.Dict() + test_dataset_path = fields.Dict() + test_dataset_count = fields.Dict() + network = fields.Dict() + optimizer = fields.Dict() + learning_rate = fields.Dict() + epoch = fields.Dict() + batch_size = fields.Dict() + loss = fields.Dict() + model_size = fields.Dict() + limit = fields.Int(validate=lambda n: 0 < n <= 100) + offset = fields.Int(validate=lambda n: 0 <= n <= 100000) + sorted_name = fields.Str() + sorted_type = fields.Str(allow_none=True) + lineage_type = fields.Str( + validate=OneOf(enum_to_list(LineageType)), + allow_none=True + ) + + @staticmethod + def check_dict_value_type(data, value_type): + """Check dict value type and int scope.""" + for key, value in data.items(): + if key == "in": + if not isinstance(value, (list, tuple)): + raise ValidationError("In operation's value must be list or tuple.") + else: + if not isinstance(value, value_type): + raise ValidationError("Wrong value type.") + if value_type is int: + if value < 0 or value > pow(2, 63) - 1: + raise ValidationError("Int value should <= pow(2, 63) - 1.") + if isinstance(value, bool): + raise ValidationError("Wrong value type.") + + @staticmethod + def check_param_value_type(data): + """Check input param's value type.""" + for key, value in data.items(): + if key == "in": + if not isinstance(value, (list, tuple)): + raise ValidationError("In operation's value must be list or tuple.") + else: + if isinstance(value, bool) or \ + (not isinstance(value, float) and not isinstance(value, int)): + raise ValidationError("Wrong value type.") + + @validates("loss") + def check_loss(self, data): + """Check loss.""" + SearchModelConditionParameter.check_param_value_type(data) + + @validates("learning_rate") + def check_learning_rate(self, data): + """Check learning_rate.""" + SearchModelConditionParameter.check_param_value_type(data) + + @validates("loss_function") + def check_loss_function(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @validates("train_dataset_path") + def check_train_dataset_path(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @validates("train_dataset_count") + def check_train_dataset_count(self, data): + SearchModelConditionParameter.check_dict_value_type(data, int) + + @validates("test_dataset_path") + def check_test_dataset_path(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @validates("test_dataset_count") + def check_test_dataset_count(self, data): + SearchModelConditionParameter.check_dict_value_type(data, int) + + @validates("network") + def check_network(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @validates("optimizer") + def check_optimizer(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @validates("epoch") + def check_epoch(self, data): + SearchModelConditionParameter.check_dict_value_type(data, int) + + @validates("batch_size") + def check_batch_size(self, data): + SearchModelConditionParameter.check_dict_value_type(data, int) + + @validates("model_size") + def check_model_size(self, data): + SearchModelConditionParameter.check_dict_value_type(data, int) + + @validates("summary_dir") + def check_summary_dir(self, data): + SearchModelConditionParameter.check_dict_value_type(data, str) + + @pre_load + def check_comparision(self, data, **kwargs): + """Check comparision for all parameters in schema.""" + for attr, condition in data.items(): + if attr in ["limit", "offset", "sorted_name", "sorted_type", "lineage_type"]: + continue + + if not isinstance(attr, str): + raise LineageParamValueError('The search attribute not supported.') + + if attr not in FIELD_MAPPING and not attr.startswith('metric_'): + raise LineageParamValueError('The search attribute not supported.') + + if not isinstance(condition, dict): + raise LineageParamTypeError("The search_condition element {} should be dict." + .format(attr)) + + for key in condition.keys(): + if key not in ["eq", "lt", "gt", "le", "ge", "in"]: + raise LineageParamValueError("The compare condition should be in " + "('eq', 'lt', 'gt', 'le', 'ge', 'in').") + + if attr.startswith('metric_'): + if len(attr) == 7: + raise LineageParamValueError( + 'The search attribute not supported.' + ) + try: + SearchModelConditionParameter.check_param_value_type(condition) + except ValidationError: + raise MindInsightException( + error=LineageErrors.LINEAGE_PARAM_METRIC_ERROR, + message=LineageErrorMsg.LINEAGE_METRIC_ERROR.value.format(attr) + ) + return data diff --git a/mindinsight/lineagemgr/common/validator/validate.py b/mindinsight/lineagemgr/common/validator/validate.py new file mode 100644 index 00000000..39435986 --- /dev/null +++ b/mindinsight/lineagemgr/common/validator/validate.py @@ -0,0 +1,395 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Validate the parameters.""" +import os + +from marshmallow import ValidationError + +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamMissingError, \ + LineageParamTypeError, LineageParamValueError, LineageDirNotExistError +from mindinsight.lineagemgr.common.log import logger as log +from mindinsight.lineagemgr.common.validator.validate_path import safe_normalize_path +from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING +from mindinsight.utils.exceptions import MindInsightException + +try: + from mindspore.nn import Cell + from mindspore.train.summary import SummaryRecord +except (ImportError, ModuleNotFoundError): + log.warning('MindSpore Not Found!') + +TRAIN_RUN_CONTEXT_ERROR_MAPPING = { + 'optimizer': LineageErrors.PARAM_OPTIMIZER_ERROR, + 'loss_fn': LineageErrors.PARAM_LOSS_FN_ERROR, + 'net_outputs': LineageErrors.PARAM_NET_OUTPUTS_ERROR, + 'train_network': LineageErrors.PARAM_TRAIN_NETWORK_ERROR, + 'train_dataset': LineageErrors.PARAM_DATASET_ERROR, + 'epoch_num': LineageErrors.PARAM_EPOCH_NUM_ERROR, + 'batch_num': LineageErrors.PARAM_BATCH_NUM_ERROR, + 'parallel_mode': LineageErrors.PARAM_TRAIN_PARALLEL_ERROR, + 'device_number': LineageErrors.PARAM_DEVICE_NUMBER_ERROR, + 'list_callback': LineageErrors.PARAM_CALLBACK_LIST_ERROR, + 'train_dataset_size': LineageErrors.PARAM_DATASET_SIZE_ERROR, +} + +SEARCH_MODEL_ERROR_MAPPING = { + 'summary_dir': LineageErrors.LINEAGE_PARAM_SUMMARY_DIR_ERROR, + 'loss_function': LineageErrors.LINEAGE_PARAM_LOSS_FUNCTION_ERROR, + 'train_dataset_path': LineageErrors.LINEAGE_PARAM_TRAIN_DATASET_PATH_ERROR, + 'train_dataset_count': LineageErrors.LINEAGE_PARAM_TRAIN_DATASET_COUNT_ERROR, + 'test_dataset_path': LineageErrors.LINEAGE_PARAM_TEST_DATASET_PATH_ERROR, + 'test_dataset_count': LineageErrors.LINEAGE_PARAM_TEST_DATASET_COUNT_ERROR, + 'network': LineageErrors.LINEAGE_PARAM_NETWORK_ERROR, + 'optimizer': LineageErrors.LINEAGE_PARAM_OPTIMIZER_ERROR, + 'learning_rate': LineageErrors.LINEAGE_PARAM_LEARNING_RATE_ERROR, + 'epoch': LineageErrors.LINEAGE_PARAM_EPOCH_ERROR, + 'batch_size': LineageErrors.LINEAGE_PARAM_BATCH_SIZE_ERROR, + 'limit': LineageErrors.PARAM_VALUE_ERROR, + 'offset': LineageErrors.PARAM_VALUE_ERROR, + 'loss': LineageErrors.LINEAGE_PARAM_LOSS_ERROR, + 'model_size': LineageErrors.LINEAGE_PARAM_MODEL_SIZE_ERROR, + 'sorted_name': LineageErrors.LINEAGE_PARAM_SORTED_NAME_ERROR, + 'sorted_type': LineageErrors.LINEAGE_PARAM_SORTED_TYPE_ERROR, + 'lineage_type': LineageErrors.LINEAGE_PARAM_LINEAGE_TYPE_ERROR +} + + +TRAIN_RUN_CONTEXT_ERROR_MSG_MAPPING = { + 'optimizer': LineageErrorMsg.PARAM_OPTIMIZER_ERROR.value, + 'loss_fn': LineageErrorMsg.PARAM_LOSS_FN_ERROR.value, + 'net_outputs': LineageErrorMsg.PARAM_NET_OUTPUTS_ERROR.value, + 'train_network': LineageErrorMsg.PARAM_TRAIN_NETWORK_ERROR.value, + 'epoch_num': LineageErrorMsg.PARAM_EPOCH_NUM_ERROR.value, + 'batch_num': LineageErrorMsg.PARAM_BATCH_NUM_ERROR.value, + 'parallel_mode': LineageErrorMsg.PARAM_TRAIN_PARALLEL_ERROR.value, + 'device_number': LineageErrorMsg.PARAM_DEVICE_NUMBER_ERROR.value, + 'list_callback': LineageErrorMsg.PARAM_CALLBACK_LIST_ERROR.value +} + +SEARCH_MODEL_ERROR_MSG_MAPPING = { + 'summary_dir': LineageErrorMsg.LINEAGE_PARAM_SUMMARY_DIR_ERROR.value, + 'loss_function': LineageErrorMsg.LINEAGE_LOSS_FUNCTION_ERROR.value, + 'train_dataset_path': LineageErrorMsg.LINEAGE_TRAIN_DATASET_PATH_ERROR.value, + 'train_dataset_count': LineageErrorMsg.LINEAGE_TRAIN_DATASET_COUNT_ERROR.value, + 'test_dataset_path': LineageErrorMsg.LINEAGE_TEST_DATASET_PATH_ERROR.value, + 'test_dataset_count': LineageErrorMsg.LINEAGE_TEST_DATASET_COUNT_ERROR.value, + 'network': LineageErrorMsg.LINEAGE_NETWORK_ERROR.value, + 'optimizer': LineageErrorMsg.LINEAGE_OPTIMIZER_ERROR.value, + 'learning_rate': LineageErrorMsg.LINEAGE_LEARNING_RATE_ERROR.value, + 'epoch': LineageErrorMsg.PARAM_EPOCH_NUM_ERROR.value, + 'batch_size': LineageErrorMsg.PARAM_BATCH_SIZE_ERROR.value, + 'limit': LineageErrorMsg.PARAM_LIMIT_ERROR.value, + 'offset': LineageErrorMsg.PARAM_OFFSET_ERROR.value, + 'loss': LineageErrorMsg.LINEAGE_LOSS_ERROR.value, + 'model_size': LineageErrorMsg.LINEAGE_MODEL_SIZE_ERROR.value, + 'sorted_name': LineageErrorMsg.LINEAGE_PARAM_SORTED_NAME_ERROR.value, + 'sorted_type': LineageErrorMsg.LINEAGE_PARAM_SORTED_TYPE_ERROR.value, + 'lineage_type': LineageErrorMsg.LINEAGE_PARAM_LINEAGE_TYPE_ERROR.value +} + + +EVAL_RUN_CONTEXT_ERROR_MAPPING = { + 'valid_dataset': LineageErrors.PARAM_DATASET_ERROR, + 'metrics': LineageErrors.PARAM_EVAL_METRICS_ERROR +} + +EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING = { + 'metrics': LineageErrorMsg.PARAM_EVAL_METRICS_ERROR.value, +} + + +def validate_int_params(int_param, param_name): + """ + Verify the parameter which type is integer valid or not. + + Args: + int_param (int): parameter that is integer, + including epoch, dataset_batch_size, step_num + param_name (str): the name of parameter, + including epoch, dataset_batch_size, step_num + + Raises: + MindInsightException: If the parameters are invalid. + """ + if not isinstance(int_param, int) or int_param <= 0 or int_param > pow(2, 63) - 1: + if param_name == 'step_num': + log.error('Invalid step_num. The step number should be a positive integer.') + raise MindInsightException(error=LineageErrors.PARAM_STEP_NUM_ERROR, + message=LineageErrorMsg.PARAM_STEP_NUM_ERROR.value) + + if param_name == 'dataset_batch_size': + log.error('Invalid dataset_batch_size. ' + 'The batch size should be a positive integer.') + raise MindInsightException(error=LineageErrors.PARAM_BATCH_SIZE_ERROR, + message=LineageErrorMsg.PARAM_BATCH_SIZE_ERROR.value) + + +def validate_network(network): + """ + Verify if the network is valid. + + Args: + network (Cell): See mindspore.nn.Cell. + + Raises: + LineageParamMissingError: If the network is None. + MindInsightException: If the network is invalid. + """ + if not network: + error_msg = "The input network for TrainLineage should not be None." + log.error(error_msg) + raise LineageParamMissingError(error_msg) + + if not isinstance(network, Cell): + log.error("Invalid network. Network should be an instance" + "of mindspore.nn.Cell.") + raise MindInsightException( + error=LineageErrors.PARAM_TRAIN_NETWORK_ERROR, + message=LineageErrorMsg.PARAM_TRAIN_NETWORK_ERROR.value + ) + + +def validate_file_path(file_path, allow_empty=False): + """ + Verify that the file_path is valid. + + Args: + file_path (str): Input file path. + allow_empty (bool): Whether file_path can be empty. + + Raises: + MindInsightException: If the parameters are invalid. + """ + try: + if allow_empty and not file_path: + return + safe_normalize_path(file_path, raise_key='dataset_path', safe_prefixes=None) + except ValidationError as error: + log.error(str(error)) + raise MindInsightException(error=LineageErrors.PARAM_FILE_PATH_ERROR, + message=str(error)) + + +def validate_train_run_context(schema, data): + """ + Validate mindspore train run_context data according to schema. + + Args: + schema (Schema): data schema. + data (dict): data to check schema. + + Raises: + MindInsightException: If the parameters are invalid. + """ + + errors = schema().validate(data) + for error_key, error_msg in errors.items(): + if error_key in TRAIN_RUN_CONTEXT_ERROR_MAPPING.keys(): + error_code = TRAIN_RUN_CONTEXT_ERROR_MAPPING.get(error_key) + if TRAIN_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key): + error_msg = TRAIN_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key) + log.error(error_msg) + raise MindInsightException(error=error_code, message=error_msg) + + +def validate_eval_run_context(schema, data): + """ + Validate mindspore evaluation job run_context data according to schema. + + Args: + schema (Schema): data schema. + data (dict): data to check schema. + + Raises: + MindInsightException: If the parameters are invalid. + """ + errors = schema().validate(data) + for error_key, error_msg in errors.items(): + if error_key in EVAL_RUN_CONTEXT_ERROR_MAPPING.keys(): + error_code = EVAL_RUN_CONTEXT_ERROR_MAPPING.get(error_key) + if EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key): + error_msg = EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key) + log.error(error_msg) + raise MindInsightException(error=error_code, message=error_msg) + + +def validate_search_model_condition(schema, data): + """ + Validate search model condition. + + Args: + schema (Schema): Data schema. + data (dict): Data to check schema. + + Raises: + MindInsightException: If the parameters are invalid. + """ + error = schema().validate(data) + for error_key in error.keys(): + if error_key in SEARCH_MODEL_ERROR_MAPPING.keys(): + error_code = SEARCH_MODEL_ERROR_MAPPING.get(error_key) + error_msg = SEARCH_MODEL_ERROR_MSG_MAPPING.get(error_key) + log.error(error_msg) + raise MindInsightException(error=error_code, message=error_msg) + + +def validate_summary_record(summary_record): + """ + Validate summary_record. + + Args: + summary_record (SummaryRecord): SummaryRecord is used to record + the summary value, and summary_record is an instance of SummaryRecord, + see mindspore.train.summary.SummaryRecord + + Raises: + MindInsightException: If the parameters are invalid. + """ + if not isinstance(summary_record, SummaryRecord): + log.error("Invalid summary_record. It should be an instance " + "of mindspore.train.summary.SummaryRecord.") + raise MindInsightException( + error=LineageErrors.PARAM_SUMMARY_RECORD_ERROR, + message=LineageErrorMsg.PARAM_SUMMARY_RECORD_ERROR.value + ) + + +def validate_raise_exception(raise_exception): + """ + Validate raise_exception. + + Args: + raise_exception (bool): decide raise exception or not, + if True, raise exception; else, catch exception and continue. + + Raises: + MindInsightException: If the parameters are invalid. + """ + if not isinstance(raise_exception, bool): + log.error("Invalid raise_exception. It should be True or False.") + raise MindInsightException( + error=LineageErrors.PARAM_RAISE_EXCEPTION_ERROR, + message=LineageErrorMsg.PARAM_RAISE_EXCEPTION_ERROR.value + ) + + +def validate_filter_key(keys): + """ + Verify the keys of filtering is valid or not. + + Args: + keys (list): The keys to get the relative lineage info. + + Raises: + LineageParamTypeError: If keys is not list. + LineageParamValueError: If the value of keys is invalid. + """ + filter_keys = [ + 'metric', 'hyper_parameters', 'algorithm', + 'train_dataset', 'model', 'valid_dataset', + 'dataset_graph' + ] + + if not isinstance(keys, list): + log.error("Keys must be list.") + raise LineageParamTypeError("Keys must be list.") + + for element in keys: + if not isinstance(element, str): + log.error("Element of keys must be str.") + raise LineageParamTypeError("Element of keys must be str.") + + if not set(keys).issubset(filter_keys): + err_msg = "Keys must be in {}.".format(filter_keys) + log.error(err_msg) + raise LineageParamValueError(err_msg) + + +def validate_condition(search_condition): + """ + Verify the param in search_condition is valid or not. + + Args: + search_condition (dict): The search condition. + + Raises: + LineageParamTypeError: If the type of the param in search_condition is invalid. + LineageParamValueError: If the value of the param in search_condition is invalid. + """ + if not isinstance(search_condition, dict): + log.error("Invalid search_condition type, it should be dict.") + raise LineageParamTypeError("Invalid search_condition type, " + "it should be dict.") + + if "limit" in search_condition: + if isinstance(search_condition.get("limit"), bool) \ + or not isinstance(search_condition.get("limit"), int): + log.error("The limit must be int.") + raise LineageParamTypeError("The limit must be int.") + + if "offset" in search_condition: + if isinstance(search_condition.get("offset"), bool) \ + or not isinstance(search_condition.get("offset"), int): + log.error("The offset must be int.") + raise LineageParamTypeError("The offset must be int.") + + if "sorted_name" in search_condition: + sorted_name = search_condition.get("sorted_name") + err_msg = "The sorted_name must be in {} or start with " \ + "`metric_`.".format(list(FIELD_MAPPING.keys())) + if not isinstance(sorted_name, str): + log.error(err_msg) + raise LineageParamValueError(err_msg) + if sorted_name not in FIELD_MAPPING and not ( + sorted_name.startswith('metric_') and len(sorted_name) > 7): + log.error(err_msg) + raise LineageParamValueError(err_msg) + + sorted_type_param = ['ascending', 'descending', None] + if "sorted_type" in search_condition: + if "sorted_name" not in search_condition: + log.error("The sorted_name have to exist when sorted_type exists.") + raise LineageParamValueError("The sorted_name have to exist when sorted_type exists.") + + if search_condition.get("sorted_type") not in sorted_type_param: + err_msg = "The sorted_type must be ascending or descending." + log.error(err_msg) + raise LineageParamValueError(err_msg) + + +def validate_path(summary_path): + """ + Verify the summary path is valid or not. + + Args: + summary_path (str): The summary path which is a dir. + + Raises: + LineageParamValueError: If the input param value is invalid. + LineageDirNotExistError: If the summary path is invalid. + """ + try: + summary_path = safe_normalize_path( + summary_path, "summary_path", None, check_absolute_path=True + ) + except ValidationError: + log.error("The summary path is invalid.") + raise LineageParamValueError("The summary path is invalid.") + if not os.path.isdir(summary_path): + log.error("The summary path does not exist or is not a dir.") + raise LineageDirNotExistError("The summary path does not exist or is not a dir.") + + return summary_path diff --git a/mindinsight/lineagemgr/common/validator/validate_path.py b/mindinsight/lineagemgr/common/validator/validate_path.py new file mode 100644 index 00000000..1f9d6f46 --- /dev/null +++ b/mindinsight/lineagemgr/common/validator/validate_path.py @@ -0,0 +1,120 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Validate the input path.""" +import os +from typing import Union, List +from marshmallow import ValidationError + + +def safe_normalize_path( + path, + raise_key, + safe_prefixes: Union[None, List[str]], + check_absolute_path=False, + allow_parent_dir=False, +): + """ + Returns safe normalized path. + + This func validates given path, and returns its normalized form. If + safe_prefixes is given, this func will check whether the path is safe. + + Note: + This func is not compatible with windows. + + Caller should check returned path to ensure safety according to + business logic. + + File scheme (rfc8089) is currently not supported. + + Args: + path (str): Path to be normalized. + + raise_key (str): The exception raise key + + safe_prefixes (list[str]): If not none, path must startswith one of the + safe_prefixes. Set this arg to [] will cause all paths considered + unsafe. Normally, prefix in this arg should end with "/". + + check_absolute_path (bool): Whether check path is absolute. + + allow_parent_dir (bool): Whether allow parent dir in path. + + Returns: + str, normalized path. + """ + normalized_path = validate_and_normalize_path( + path, + raise_key=raise_key, + check_absolute_path=check_absolute_path, + allow_parent_dir=allow_parent_dir, + ) + + if safe_prefixes is None: + return normalized_path + + normalized_str = str(normalized_path) + for prefix in safe_prefixes: + if normalized_str.startswith(prefix): + return normalized_path + + raise ValidationError({raise_key: {"The path is invalid!"}}) + + +def validate_and_normalize_path( + path, + raise_key, + check_absolute_path=False, + allow_parent_dir=False, +): + """ + Validates path and returns its normalized form. + + If path has a valid scheme, treat path as url, otherwise consider path a + unix local path. + + Note: + File scheme (rfc8089) is currently not supported. + + Args: + path (str): Path to be normalized. + raise_key (str): The exception raise key. + check_absolute_path (bool): Whether check path scheme is supported. + allow_parent_dir (bool): Whether allow parent dir in path. + + + Returns: + str, normalized path. + """ + if not path: + raise ValidationError({raise_key: {"The path is invalid!"}}) + + path_str = str(path) + if not allow_parent_dir: + path_components = path_str.split("/") + if ".." in path_components: + raise ValidationError({raise_key: {"The path is invalid!"}}) + + # path does not have valid schema, treat it as unix local path. + if check_absolute_path: + if not path_str.startswith("/"): + raise ValidationError({raise_key: {"The path is invalid!"}}) + try: + # most unix systems allow + normalized_path = os.path.realpath(path) + except ValueError: + raise ValidationError({raise_key: {"The path is invalid!"}}) + + return normalized_path diff --git a/mindinsight/lineagemgr/querier/__init__.py b/mindinsight/lineagemgr/querier/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/mindinsight/lineagemgr/querier/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/querier/querier.py b/mindinsight/lineagemgr/querier/querier.py new file mode 100644 index 00000000..407ab41a --- /dev/null +++ b/mindinsight/lineagemgr/querier/querier.py @@ -0,0 +1,446 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file is used to define lineage info querier.""" +import enum +import functools +import operator +import os + +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageParamTypeError, LineageSummaryAnalyzeException, \ + LineageEventNotExistException, LineageQuerierParamException, \ + LineageSummaryParseException, LineageEventFieldNotExistException +from mindinsight.lineagemgr.common.log import logger +from mindinsight.lineagemgr.querier.query_model import LineageObj, FIELD_MAPPING +from mindinsight.lineagemgr.summary.lineage_summary_analyzer import \ + LineageSummaryAnalyzer + + +@enum.unique +class ConditionParam(enum.Enum): + """ + Filtering and sorting field names. + + `LIMIT` represents the number of lineage info per page. `OFFSET` represents + page number. `SORTED_NAME` means to sort by this field. `SORTED_TYPE` means + ascending or descending. + """ + LIMIT = 'limit' + OFFSET = 'offset' + SORTED_NAME = 'sorted_name' + SORTED_TYPE = 'sorted_type' + LINEAGE_TYPE = 'lineage_type' + + @classmethod + def is_condition_type(cls, value): + """ + Judge that the input param is one of field names in the class. + + Args: + value (str): The input field name. + + Returns: + bool, `True` if the input field name in the class, else `False`. + """ + return value in cls._value2member_map_ + + +@enum.unique +class ExpressionType(enum.Enum): + """ + Filter condition name definition. + + `EQ` means `==`. `LT` means `<`. `GT` means `>`. `LE` means `<=`. `GE` means + `>=`. `IN` means filter value in the specified list. + """ + EQ = 'eq' + LT = 'lt' + GT = 'gt' + LE = 'le' + GE = 'ge' + IN = 'in' + + @classmethod + def is_valid_exp(cls, key): + """ + Judge that the input param is one of filter condition names in the class. + + Args: + key (str): The input filter condition name. + + Returns: + bool, `True` if the input filter condition name in the class, + else `False`. + """ + return key in cls._value2member_map_ + + @classmethod + def is_match(cls, except_key, except_value, actual_value): + """ + Determine whether the value meets the expected requirement. + + Args: + except_key (str): The expression key. + except_value (Union[str, int, float, list, tuple]): The expected + value. + actual_value (Union[str, int, float]): The actual value. + + Returns: + bool, `True` if the actual value meets the expected requirement, + else `False`. + """ + if actual_value is None and except_key in [cls.LT.value, cls.GT.value, + cls.LE.value, cls.GE.value]: + return False + + if except_key == cls.IN.value: + state = operator.contains(except_value, actual_value) + else: + state = getattr(operator, except_key)(actual_value, except_value) + return state + + +@enum.unique +class LineageFilterKey(enum.Enum): + """Summary lineage information filter key.""" + METRIC = 'metric' + HYPER_PARAM = 'hyper_parameters' + ALGORITHM = 'algorithm' + TRAIN_DATASET = 'train_dataset' + VALID_DATASET = 'valid_dataset' + MODEL = 'model' + DATASET_GRAPH = 'dataset_graph' + + @classmethod + def is_valid_filter_key(cls, key): + """ + Judge that the input param is one of field names in the class. + + Args: + key (str): The input field name. + + Returns: + bool, `True` if the input field name in the class, else `False`. + """ + return key in cls._value2member_map_ + + @classmethod + def get_key_list(cls): + """ + Get the filter key name list. + + Returns: + list[str], the filter key name list. + """ + return [member.value for member in cls] + + +@enum.unique +class LineageType(enum.Enum): + """Lineage search type.""" + DATASET = 'dataset' + MODEL = 'model' + + +class Querier: + """ + The querier of model lineage information. + + The class provides model lineage information query function. The information + includes hyper parameters, train dataset, algorithm, model information, + metric, valid dataset, etc. + + The class also provides search and sorting capabilities about model lineage + information. You can search and sort by the specified condition. + The condition explain in `ConditionParam` and `ExpressionType` class. + See the method `filter_summary_lineage` for supported fields. + + Args: + summary_path (Union[str, list[str]]): The single summary log path or + a list of summary log path. + + Raises: + LineageParamTypeError: If the input parameter type is invalid. + LineageQuerierParamException: If the input parameter value is invalid. + LineageSummaryParseException: If all summary logs parsing failed. + """ + def __init__(self, summary_path): + self._lineage_objects = [] + self._index_map = {} + self._parse_failed_paths = [] + self._parse_summary_logs(summary_path) + self._size = len(self._lineage_objects) + + def get_summary_lineage(self, summary_dir=None, filter_keys=None): + """ + Get summary lineage information. + + If a summary dir is specified, the special summary lineage information + will be found. If the summary dir is `None`, all summary lineage + information will be found. + + Returns the content corresponding to the specified field in the filter + key. The contents of the filter key include `metric`, `hyper_parameters`, + `algorithm`, `train_dataset`, `valid_dataset` and `model`. You can + specify multiple filter keys in the `filter_keys`. If the parameter is + `None`, complete information will be returned. + + Args: + summary_dir (Union[str, None]): Summary log dir. Default: None. + filter_keys (Union[list[str], None]): Filter keys. Default: None. + + Returns: + list[dict], summary lineage information. + """ + self._parse_fail_summary_logs() + + if filter_keys is None: + filter_keys = LineageFilterKey.get_key_list() + else: + for key in filter_keys: + if not LineageFilterKey.is_valid_filter_key(key): + raise LineageQuerierParamException( + filter_keys, 'The filter key {} is invalid.'.format(key) + ) + + if summary_dir is None: + result = [ + item.get_summary_info(filter_keys) for item in self._lineage_objects + ] + else: + index = self._index_map.get(summary_dir) + if index is None: + raise LineageQuerierParamException( + 'summary_dir', + 'Summary dir {} does not exist.'.format(summary_dir) + ) + lineage_obj = self._lineage_objects[index] + result = [lineage_obj.get_summary_info(filter_keys)] + return result + + def filter_summary_lineage(self, condition=None): + """ + Filter and sort lineage information based on the specified condition. + + See `ConditionType` and `ExpressionType` class for the rule of filtering + and sorting. The filtering and sorting fields are defined in + `FIELD_MAPPING` or prefixed with `metric_`. + + If the condition is `None`, all model lineage information will be + returned. + + Args: + condition (Union[dict, None]): Filter and sort condition. + Default: None. + + Returns: + dict, filtered and sorted model lineage information. + """ + def _filter(lineage_obj: LineageObj): + for condition_key, condition_value in condition.items(): + if ConditionParam.is_condition_type(condition_key): + continue + if self._is_valid_field(condition_key): + raise LineageQuerierParamException( + 'condition', + 'The field {} not supported'.format(condition_key) + ) + + value = lineage_obj.get_value_by_key(condition_key) + for exp_key, exp_value in condition_value.items(): + if not ExpressionType.is_valid_exp(exp_key): + raise LineageQuerierParamException( + 'condition', + 'The expression {} not supported.'.format(exp_key) + ) + if not ExpressionType.is_match(exp_key, exp_value, value): + return False + return True + + def _cmp(obj1: LineageObj, obj2: LineageObj): + value1 = obj1.get_value_by_key(sorted_name) + value2 = obj2.get_value_by_key(sorted_name) + + if value1 is None and value2 is None: + cmp_result = 0 + elif value1 is None: + cmp_result = -1 + elif value2 is None: + cmp_result = 1 + else: + cmp_result = (value1 > value2) - (value1 < value2) + + return cmp_result + + self._parse_fail_summary_logs() + + if condition is None: + condition = {} + result = list(filter(_filter, self._lineage_objects)) + + if ConditionParam.SORTED_NAME.value in condition: + sorted_name = condition.get(ConditionParam.SORTED_NAME.value) + if self._is_valid_field(sorted_name): + raise LineageQuerierParamException( + 'condition', + 'The sorted name {} not supported.'.format(sorted_name) + ) + sorted_type = condition.get(ConditionParam.SORTED_TYPE.value) + reverse = sorted_type == 'descending' + result = sorted( + result, key=functools.cmp_to_key(_cmp), reverse=reverse + ) + + offset_result = self._handle_limit_and_offset(condition, result) + + search_type = condition.get(ConditionParam.LINEAGE_TYPE.value) + lineage_info = { + 'object': [ + item.to_dataset_lineage_dict() if search_type == LineageType.DATASET.value + else item.to_filtration_dict() for item in offset_result + ], + 'count': len(result) + } + + return lineage_info + + def _is_valid_field(self, field_name): + """ + Check if field name is valid. + + Args: + field_name (str): Field name. + + Returns: + bool, `True` if the field name is valid, else `False`. + """ + return field_name not in FIELD_MAPPING and not field_name.startswith('metric_') + + def _handle_limit_and_offset(self, condition, result): + """ + Handling the condition of `limit` and `offset`. + + Args: + condition (dict): Filter and sort condition. + result (list[LineageObj]): Filtered and sorted result. + + Returns: + list[LineageObj], paginated result. + """ + offset = 0 + limit = 10 + if ConditionParam.OFFSET.value in condition: + offset = condition.get(ConditionParam.OFFSET.value) + if ConditionParam.LIMIT.value in condition: + limit = condition.get(ConditionParam.LIMIT.value) + if ConditionParam.OFFSET.value not in condition \ + and ConditionParam.LIMIT.value not in condition: + offset_result = result + else: + offset_result = result[offset * limit: limit * (offset + 1)] + return offset_result + + def _parse_summary_logs(self, summary_path): + """ + Parse summary logs. + + Args: + summary_path (Union[str, list[str]]): The single summary log path or + a list of summary log path. + """ + if not summary_path: + raise LineageQuerierParamException( + 'summary_path', 'The summary path is empty.' + ) + if isinstance(summary_path, str): + self._parse_summary_log(summary_path, 0) + elif isinstance(summary_path, list): + index = 0 + for path in summary_path: + parse_result = self._parse_summary_log(path, index) + if parse_result: + index += 1 + else: + raise LineageParamTypeError('Summary path is not str or list.') + + if self._parse_failed_paths: + logger.info('Parse failed paths: %s', str(self._parse_failed_paths)) + + if not self._lineage_objects: + raise LineageSummaryParseException() + + def _parse_summary_log(self, log_path, index: int, is_save_fail_path=True): + """ + Parse the single summary log. + + Args: + log_path (str): The single summary log path. + index (int): TrainInfo instance index in the train info list. + is_save_fail_path (bool): Set whether to save the failed summary + path. Default: True. + + Returns: + bool, `True` if parse summary log success, else `False`. + """ + log_dir = os.path.dirname(log_path) + try: + lineage_info = LineageSummaryAnalyzer.get_summary_infos(log_path) + lineage_obj = LineageObj( + log_dir, + train_lineage=lineage_info.train_lineage, + evaluation_lineage=lineage_info.eval_lineage, + dataset_graph=lineage_info.dataset_graph + ) + self._lineage_objects.append(lineage_obj) + self._add_dataset_mark() + self._index_map[log_dir] = index + return True + except (LineageSummaryAnalyzeException, + LineageEventNotExistException, + LineageEventFieldNotExistException): + if is_save_fail_path: + self._parse_failed_paths.append(log_path) + return False + + def _parse_fail_summary_logs(self): + """Parse fail summary logs.""" + if self._parse_failed_paths: + failed_paths = [] + for path in self._parse_failed_paths: + parse_result = self._parse_summary_log(path, self._size, False) + if parse_result: + self._size += 1 + else: + failed_paths.append(path) + self._parse_failed_paths = failed_paths + + def _add_dataset_mark(self): + """Add dataset mark into LineageObj.""" + # give a dataset mark for each dataset graph in lineage information + marked_dataset_group = {'1': None} + for lineage in self._lineage_objects: + dataset_mark = '0' + for dataset_graph_mark, marked_dataset_graph in marked_dataset_group.items(): + if marked_dataset_graph == lineage.dataset_graph: + dataset_mark = dataset_graph_mark + break + # if no matched, add the new dataset graph into group + if dataset_mark == '0': + dataset_mark = str(int(max(marked_dataset_group.keys())) + 1) + marked_dataset_group.update({ + dataset_mark: + lineage.dataset_graph + }) + lineage.dataset_mark = dataset_mark diff --git a/mindinsight/lineagemgr/querier/query_model.py b/mindinsight/lineagemgr/querier/query_model.py new file mode 100644 index 00000000..0ca4be64 --- /dev/null +++ b/mindinsight/lineagemgr/querier/query_model.py @@ -0,0 +1,344 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""This file is used to define lineage info model.""" +import json +from collections import namedtuple + +from google.protobuf.json_format import MessageToDict + +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageEventFieldNotExistException, LineageEventNotExistException +from mindinsight.lineagemgr.summary._summary_adapter import organize_graph + +Field = namedtuple('Field', ['base_name', 'sub_name']) +FIELD_MAPPING = { + "summary_dir": Field('summary_dir', None), + "loss_function": Field("hyper_parameters", 'loss_function'), + "train_dataset_path": Field('train_dataset', 'train_dataset_path'), + "train_dataset_count": Field("train_dataset", 'train_dataset_size'), + "test_dataset_path": Field('valid_dataset', 'valid_dataset_path'), + "test_dataset_count": Field('valid_dataset', 'valid_dataset_size'), + "network": Field('algorithm', 'network'), + "optimizer": Field('hyper_parameters', 'optimizer'), + "learning_rate": Field('hyper_parameters', 'learning_rate'), + "epoch": Field('hyper_parameters', 'epoch'), + "batch_size": Field('hyper_parameters', 'batch_size'), + "loss": Field('algorithm', 'loss'), + "model_size": Field('model', 'size'), + "dataset_mark": Field('dataset_mark', None), +} + + +class LineageObj: + """ + Lineage information class. + + An instance of the class hold lineage information for a training session. + + Args: + summary_dir (str): Summary log dir. + kwargs (dict): Params to init the instance. + + - train_lineage (Event): Train lineage object. + + - evaluation_lineage (Event): Evaluation lineage object. + + - dataset_graph (Event): Dataset graph object. + + Raises: + LineageEventNotExistException: If train and evaluation event not exist. + LineageEventFieldNotExistException: If the special event field not exist. + """ + _name_train_lineage = 'train_lineage' + _name_evaluation_lineage = 'evaluation_lineage' + _name_summary_dir = 'summary_dir' + _name_metric = 'metric' + _name_hyper_parameters = 'hyper_parameters' + _name_algorithm = 'algorithm' + _name_train_dataset = 'train_dataset' + _name_model = 'model' + _name_valid_dataset = 'valid_dataset' + _name_dataset_graph = 'dataset_graph' + _name_dataset_mark = 'dataset_mark' + + def __init__(self, summary_dir, **kwargs): + self._lineage_info = { + self._name_summary_dir: summary_dir + } + train_lineage = kwargs.get('train_lineage') + evaluation_lineage = kwargs.get('evaluation_lineage') + dataset_graph = kwargs.get('dataset_graph') + if not any([train_lineage, evaluation_lineage, dataset_graph]): + raise LineageEventNotExistException() + self._parse_train_lineage(train_lineage) + self._parse_evaluation_lineage(evaluation_lineage) + self._parse_dataset_graph(dataset_graph) + self._filtration_result = self._organize_filtration_result() + + @property + def summary_dir(self): + """ + Get summary log dir. + + Returns: + str, the summary log dir. + """ + return self._lineage_info.get(self._name_summary_dir) + + @property + def metric(self): + """ + Get metric information. + + Returns: + dict, the metric information. + """ + return self._lineage_info.get(self._name_metric) + + @property + def hyper_parameters(self): + """ + Get hyperparameters. + + Returns: + dict, the hyperparameters. + """ + return self._lineage_info.get(self._name_hyper_parameters) + + @property + def algorithm(self): + """ + Get algorithm. + + Returns: + dict, the algorithm. + """ + return self._lineage_info.get(self._name_algorithm) + + @property + def train_dataset(self): + """ + Get train dataset information. + + Returns: + dict, the train dataset information. + """ + return self._lineage_info.get(self._name_train_dataset) + + @property + def model(self): + """ + Get model information. + + Returns: + dict, the model information. + """ + return self._lineage_info.get(self._name_model) + + @property + def valid_dataset(self): + """ + Get valid dataset information. + + Returns: + dict, the valid dataset information. + """ + return self._lineage_info.get(self._name_valid_dataset) + + @property + def dataset_graph(self): + """ + Get dataset_graph. + + Returns: + dict, the dataset graph information. + """ + return self._lineage_info.get(self._name_dataset_graph) + + @property + def dataset_mark(self): + """ + Get dataset_mark. + + Returns: + dict, the dataset mark information. + """ + return self._lineage_info.get(self._name_dataset_mark) + + @dataset_mark.setter + def dataset_mark(self, dataset_mark): + """ + Set dataset mark. + + Args: + dataset_mark (int): Dataset mark. + """ + self._lineage_info[self._name_dataset_mark] = dataset_mark + # update dataset_mark into filtration result + self._filtration_result[self._name_dataset_mark] = dataset_mark + + def get_summary_info(self, filter_keys: list): + """ + Get the summary lineage information. + + Returns the content corresponding to the specified field in the filter + key. The contents of the filter key include `metric`, `hyper_parameters`, + `algorithm`, `train_dataset`, `valid_dataset` and `model`. You can + specify multiple filter keys in the `filter_keys` + + Args: + filter_keys (list): Filter keys. + + Returns: + dict, the summary lineage information. + """ + result = { + self._name_summary_dir: self.summary_dir, + } + + for key in filter_keys: + result[key] = getattr(self, key) + return result + + def to_filtration_dict(self): + """ + Returns the lineage information required by filtering interface. + + Returns: + dict, the lineage information required by filtering interface. + """ + return self._filtration_result + + def to_dataset_lineage_dict(self): + """ + Returns the dataset part lineage information. + + Returns: + dict, the dataset lineage information. + """ + dataset_lineage = { + key: self._filtration_result.get(key) + for key in [self._name_summary_dir, self._name_dataset_graph] + } + + return dataset_lineage + + def get_value_by_key(self, key): + """ + Get the value based on the key in `FIELD_MAPPING` or the key prefixed with `metric_`. + + Args: + key (str): The key in `FIELD_MAPPING` or prefixed with `metric_`. + + Returns: + object, the value. + """ + if key.startswith('metric_'): + metric_key = key.split('_', 1)[1] + metric = self._filtration_result.get(self._name_metric) + if metric: + return metric.get(metric_key) + return self._filtration_result.get(key) + + def _organize_filtration_result(self): + """ + Organize filtration result. + + Returns: + dict, the filtration result. + """ + result = {} + for key, field in FIELD_MAPPING.items(): + if field.base_name is not None: + base_attr = getattr(self, field.base_name) + result[key] = base_attr.get(field.sub_name) \ + if field.sub_name else base_attr + # add metric into filtration result + result[self._name_metric] = self.metric + # add dataset_graph into filtration result + result[self._name_dataset_graph] = getattr(self, self._name_dataset_graph) + return result + + def _parse_train_lineage(self, train_lineage): + """ + Parse train lineage. + + Args: + train_lineage (Event): Train lineage. + """ + if train_lineage is None: + self._lineage_info[self._name_model] = {} + self._lineage_info[self._name_algorithm] = {} + self._lineage_info[self._name_hyper_parameters] = {} + self._lineage_info[self._name_train_dataset] = {} + return + + event_dict = MessageToDict( + train_lineage, preserving_proto_field_name=True + ) + train_dict = event_dict.get(self._name_train_lineage) + if train_dict is None: + raise LineageEventFieldNotExistException( + self._name_train_lineage + ) + + # when MessageToDict is converted to dict, int64 type is converted + # to string, so we convert it to an int in python + if train_dict.get(self._name_model): + model_size = train_dict.get(self._name_model).get('size') + if model_size: + train_dict[self._name_model]['size'] = int(model_size) + + self._lineage_info.update(**train_dict) + + def _parse_evaluation_lineage(self, evaluation_lineage): + """ + Parse evaluation lineage. + + Args: + evaluation_lineage (Event): Evaluation lineage. + """ + if evaluation_lineage is None: + self._lineage_info[self._name_metric] = {} + self._lineage_info[self._name_valid_dataset] = {} + return + + event_dict = MessageToDict( + evaluation_lineage, preserving_proto_field_name=True + ) + evaluation_dict = event_dict.get(self._name_evaluation_lineage) + if evaluation_dict is None: + raise LineageEventFieldNotExistException( + self._name_evaluation_lineage + ) + self._lineage_info.update(**evaluation_dict) + metric = self._lineage_info.get(self._name_metric) + self._lineage_info[self._name_metric] = json.loads(metric) if metric else {} + + def _parse_dataset_graph(self, dataset_graph): + """ + Parse dataset graph. + + Args: + dataset_graph (Event): Dataset graph. + """ + if dataset_graph is None: + self._lineage_info[self._name_dataset_graph] = {} + else: + # convert message to dict + event_dict = organize_graph(dataset_graph.dataset_graph) + if event_dict is None: + raise LineageEventFieldNotExistException(self._name_evaluation_lineage) + self._lineage_info[self._name_dataset_graph] = event_dict if event_dict else {} diff --git a/mindinsight/lineagemgr/summary/__init__.py b/mindinsight/lineagemgr/summary/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/mindinsight/lineagemgr/summary/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/lineagemgr/summary/_summary_adapter.py b/mindinsight/lineagemgr/summary/_summary_adapter.py new file mode 100644 index 00000000..b2b78131 --- /dev/null +++ b/mindinsight/lineagemgr/summary/_summary_adapter.py @@ -0,0 +1,293 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The converter between proto format event of lineage and dict.""" +import time + +from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError +from mindinsight.lineagemgr.common.log import logger as log + + +def package_dataset_graph(graph): + """ + Package dataset graph. + + Args: + graph (dict): Dataset graph. + + Returns: + Event, the proto message event contains dataset graph. + """ + dataset_graph_event = Event() + dataset_graph_event.wall_time = time.time() + + dataset_graph = dataset_graph_event.dataset_graph + if "children" in graph: + children = graph.pop("children") + if children: + _package_children(children=children, message=dataset_graph) + _package_current_dataset(operation=graph, message=dataset_graph) + + return dataset_graph_event + + +def _package_children(children, message): + """ + Package children in dataset operation. + + Args: + children (list[dict]): Child operations. + message (DatasetGraph): Children proto message. + """ + for child in children: + if child: + child_graph_message = getattr(message, "children").add() + grandson = child.pop("children") + if grandson: + _package_children(children=grandson, message=child_graph_message) + # package other parameters + _package_current_dataset(operation=child, message=child_graph_message) + + +def _package_current_dataset(operation, message): + """ + Package operation parameters in event message. + + Args: + operation (dict): Operation dict. + message (Operation): Operation proto message. + """ + for key, value in operation.items(): + if key == "operations": + for operator in value: + _package_enhancement_operation( + operator, + message.operations.add() + ) + elif key == "sampler": + _package_enhancement_operation( + value, + message.sampler + ) + else: + _package_parameter(key, value, message.parameter) + + +def _package_enhancement_operation(operation, message): + """ + Package enhancement operation in MapDataset. + + Args: + operation (dict): Enhancement operation. + message (Operation): Enhancement operation proto message. + """ + + for key, value in operation.items(): + if isinstance(value, list): + if all(isinstance(ele, int) for ele in value): + message.size.extend(value) + else: + message.weights.extend(value) + else: + _package_parameter(key, value, message.operationParam) + + +def _package_parameter(key, value, message): + """ + Package parameters in operation. + + Args: + key (str): Operation name. + value (Union[str, bool, int, float, list, None]): Operation args. + message (OperationParameter): Operation proto message. + """ + if isinstance(value, str): + message.mapStr[key] = value + elif isinstance(value, bool): + message.mapBool[key] = value + elif isinstance(value, int): + message.mapInt[key] = value + elif isinstance(value, float): + message.mapDouble[key] = value + elif isinstance(value, list) and key != "operations": + if value: + replace_value_list = list(map(lambda x: "" if x is None else x, value)) + message.mapStrList[key].strValue.extend(replace_value_list) + elif value is None: + message.mapStr[key] = "None" + else: + error_msg = "Parameter {} is not supported " \ + "in event package.".format(key) + log.error(error_msg) + raise LineageParamTypeError(error_msg) + + +def organize_graph(graph_message): + """ + Convert a dataset graph to its dict format. + + Args: + graph_message (DatasetGraph): Graph event message. + + Returns: + dict, dataset graph. + """ + result = {} + # update current dataset graph dict + result.update(_organize_current_dataset( + parameter=getattr(graph_message, 'parameter'), + operations=getattr(graph_message, 'operations'), + sampler=getattr(graph_message, 'sampler') + )) + # update children dataset graph dict + result.update( + _organize_children(getattr(graph_message, 'children')) + ) + + return result + + +def _organize_children(children_message): + """ + Convert children message to its dict format. + + Args: + children_message (list[DatasetGraph]): Children message. + + Returns: + dict, children dict of dataset graph. + """ + children_list = [] + children_dict = {'children': children_list} + if children_message: + for child_event in children_message: + child_dict = {} + # update current dataset to child + child_dict.update( + _organize_current_dataset( + parameter=getattr(child_event, 'parameter'), + operations=getattr(child_event, 'operations'), + sampler=getattr(child_event, 'sampler') + ) + ) + # update child's children + child_dict.update( + _organize_children(getattr(child_event, 'children')) + ) + children_list.append(child_dict) + children_dict['children'] = children_list + + return children_dict + + +def _organize_current_dataset(parameter, operations, sampler): + """ + Convert current dataset message to its dict format. + + Note: + Current dataset message include parameter, operations, + sampler message of dataset graph event. + + Args: + parameter (OperationParameter): Parameter message. + operations (Operation): Operations message. + sampler (Operation): Sampler message. + + Returns: + dict, current dataset. + """ + current_dataset = {} + if parameter: + current_dataset.update( + _organize_parameter(parameter) + ) + if operations: + operation_list = [] + for operation in operations: + operation_list.append( + _organize_operation(operation) + ) + current_dataset.update( + {'operations': operation_list} + ) + if sampler: + if _organize_operation(sampler): + current_dataset.update({ + 'sampler': + _organize_operation(sampler) + }) + return current_dataset + + +def _organize_operation(operation): + """ + Convert operation message to its dict format. + + Args: + operation (Operation): Operation message. + + Returns: + dict, operation. + """ + operation_dict = {} + operation_dict.update(_organize_parameter(getattr(operation, 'operationParam'))) + tmp_list = [] + repeated_keys = ['size', 'weights'] + for key in repeated_keys: + for str_ele in getattr(operation, key): + tmp_list.append(str_ele) + dict() + if tmp_list: + operation_dict.update({key: tmp_list}) + return operation_dict + + +def _organize_parameter(parameter): + """ + Convert operation parameter message to its dict format. + + Args: + parameter (OperationParameter): Operation parameter message. + + Returns: + dict, operation parameter. + """ + parameter_result = dict() + parameter_keys = [ + 'mapStr', + 'mapBool', + 'mapInt', + 'mapDouble', + ] + for parameter_key in parameter_keys: + base_attr = getattr(parameter, parameter_key) + parameter_value = dict(base_attr) + # convert str 'None' to None + for key, value in parameter_value.items(): + if value == 'None': + parameter_value[key] = None + parameter_result.update(parameter_value) + # drop `mapStrList` and `strValue` keys in result parameter + str_list_para = dict(getattr(parameter, 'mapStrList')) + result_str_list_para = dict() + for key, value in str_list_para.items(): + str_list_para_list = list() + for str_ele in getattr(value, 'strValue'): + str_list_para_list.append(str_ele) + str_list_para_list = list(map(lambda x: None if x == '' else x, str_list_para_list)) + result_str_list_para[key] = str_list_para_list + parameter_result.update(result_str_list_para) + + return parameter_result diff --git a/mindinsight/lineagemgr/summary/event_writer.py b/mindinsight/lineagemgr/summary/event_writer.py new file mode 100644 index 00000000..5579cb97 --- /dev/null +++ b/mindinsight/lineagemgr/summary/event_writer.py @@ -0,0 +1,95 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Event writer to record lineage message to summary log.""" +import os +import stat +import struct + +from mindinsight.datavisual.utils import crc32 + +KMASKDELTA = 0xa282ead8 + + +class EventWriter: + """ + Lineage summary record. + Recording train lineage and evaluation lineage to summary log. + + Args: + file_path (str): Summary log path. + override (bool): If override the summary log exist. + + Raises: + IOError: Write to summary log failed or file_path is a dir. + + Examples: + >>> content = b'\x01\x02\x03\x04' + >>> event_writer = EventWriter("./test.log", True) + >>> event_writer.write_event_to_file(content) + """ + def __init__(self, file_path, override=False): + """ + Init EventWriter, get the type of writing. + + Args: + file_path (str): The file path to writing. + override (bool): The type of writing. + """ + if os.path.exists(file_path): + if not os.path.isfile(file_path): + raise IOError("The file_path is not a normal file.") + + self.file_path = file_path + if override: + self.write_type = 'wb' + else: + self.write_type = 'ab' + + def write_event_to_file(self, content): + """ + Write event to file. + + Args: + content (bytes): Content to write. + """ + length = struct.pack("> 15) | (crc_value << 17)) & mask + crc_value = (crc_value + KMASKDELTA) & mask + + return struct.pack(">> summary_log_path = "./test.log" + >>> train_lineage = {"train_network": "ResNet"} + >>> lineage_summary = LineageSummary(summary_log_path=summary_log_path) + >>> lineage_summary.record_train_lineage(train_lineage) + """ + def __init__(self, summary_log_path=None, override=False): + self.event_writer = EventWriter(summary_log_path, override) + + @staticmethod + def package_train_message(run_context_args): + """ + Package train message. + + Args: + run_context_args (dict): The train lineage info to log. + + Returns: + Event, the proto message event contains train lineage. + """ + train_lineage_event = Event() + train_lineage_event.wall_time = time.time() + + # Init train_lineage message. + train_lineage = train_lineage_event.train_lineage + + # Construct algorithm message. + if run_context_args.get('train_network') is not None: + train_lineage.algorithm.network = run_context_args.get('train_network') + if run_context_args.get('loss') is not None: + train_lineage.algorithm.loss = run_context_args.get('loss') + # Construct hyper_parameters message. + LineageSummary.construct_hyper_parameters(train_lineage, run_context_args) + # Construct train_dataset message. + if run_context_args.get('train_dataset_path') is not None: + train_lineage.train_dataset.train_dataset_path = run_context_args.get( + 'train_dataset_path') + if run_context_args.get('train_dataset_size') is not None: + train_lineage.train_dataset.train_dataset_size = run_context_args.get( + 'train_dataset_size') + # Construct model message + if run_context_args.get('model_path') is not None: + train_lineage.model.path = run_context_args.get('model_path') + if run_context_args.get('model_size') is not None: + train_lineage.model.size = run_context_args.get('model_size') + + return train_lineage_event + + @staticmethod + def construct_hyper_parameters(train_lineage, run_context_args): + """ + Construct hyper-parameters. + + Args: + train_lineage (TrainLineage): TrainLineage defined in protobuf. + run_context_args (dict): The run_context_args. + """ + if run_context_args.get('learning_rate') is not None: + train_lineage.hyper_parameters.learning_rate = run_context_args.get('learning_rate') + if run_context_args.get('optimizer') is not None: + train_lineage.hyper_parameters.optimizer = run_context_args.get('optimizer') + if run_context_args.get('loss_function') is not None: + train_lineage.hyper_parameters.loss_function = run_context_args.get('loss_function') + if run_context_args.get('epoch') is not None: + train_lineage.hyper_parameters.epoch = run_context_args.get('epoch') + if run_context_args.get('parallel_mode') is not None: + train_lineage.hyper_parameters.parallel_mode = run_context_args.get('parallel_mode') + if run_context_args.get('device_num') is not None: + train_lineage.hyper_parameters.device_num = run_context_args.get('device_num') + if run_context_args.get('batch_size') is not None: + train_lineage.hyper_parameters.batch_size = run_context_args.get('batch_size') + + def record_train_lineage(self, run_context_args): + """ + Record train_lineage to summary log. + + Args: + run_context_args (dict): The train lineage info to log. + """ + self.event_writer.write_event_to_file( + LineageSummary.package_train_message(run_context_args).SerializeToString() + ) + + @staticmethod + def package_evaluation_message(run_context_args): + """ + Record evaluation lineage. + + Args: + run_context_args (dict): The evaluation lineage info to log. + + Returns: + Event, the proto message event contains evaluation lineage. + """ + train_lineage_event = Event() + train_lineage_event.wall_time = time.time() + + # Init evaluation_lineage message. + evaluation_lineage = train_lineage_event.evaluation_lineage + if run_context_args.get('metrics') is not None: + evaluation_lineage.metric = run_context_args.get('metrics') + # Construct valid_dataset message. + if run_context_args.get('valid_dataset_path') is not None: + evaluation_lineage.valid_dataset.valid_dataset_path = \ + run_context_args.get('valid_dataset_path') + if run_context_args.get('valid_dataset_size') is not None: + evaluation_lineage.valid_dataset.valid_dataset_size = \ + run_context_args.get('valid_dataset_size') + + return train_lineage_event + + def record_evaluation_lineage(self, run_context_args): + """ + Record evaluation_lineage to sumamry log. + + Args: + run_context_args (dict): The evaluation lineage info to log. + + """ + self.event_writer.write_event_to_file( + LineageSummary.package_evaluation_message(run_context_args).SerializeToString() + ) + + def record_dataset_graph(self, dataset_graph): + """ + Record dataset graph to summary log. + + Args: + dataset_graph (dict): The dataset graph to log. + """ + self.event_writer.write_event_to_file( + package_dataset_graph(dataset_graph).SerializeToString() + ) diff --git a/mindinsight/scripts/__init__.py b/mindinsight/scripts/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/mindinsight/scripts/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/scripts/start.py b/mindinsight/scripts/start.py new file mode 100644 index 00000000..3ce62b83 --- /dev/null +++ b/mindinsight/scripts/start.py @@ -0,0 +1,211 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Start mindinsight service.""" + +import os +import argparse +from importlib import import_module + +import psutil + +from mindinsight.conf import settings +from mindinsight.utils.command import BaseCommand +from mindinsight.utils.hook import HookUtils +from mindinsight.utils.hook import init +from mindinsight.utils.exceptions import PortNotAvailableError + + +class ConfigAction(argparse.Action): + """Config action class definition.""" + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + config = values + if not config: + setattr(namespace, self.dest, config) + + # python:full.path.for.config.module + if config.startswith('python:'): + config = config[len('python:'):] + try: + import_module(config) + except ModuleNotFoundError: + parser.error(f'{option_string} {config} not exists') + + config = f'python:{config}' + else: + # file:/full/path/for/config/module.py + if config.startswith('file:'): + config = config[len('file:'):] + + if config.startswith('~'): + config = os.path.realpath(os.path.expanduser(config)) + + if not config.startswith('/'): + config = os.path.realpath(os.path.join(os.getcwd(), config)) + + if not os.path.exists(config): + parser.error(f'{option_string} {config} not exists') + + if not os.access(config, os.R_OK): + parser.error(f'{option_string} {config} not accessible') + + config = f'file:{config}' + + setattr(namespace, self.dest, config) + + +class WorkspaceAction(argparse.Action): + """Workspace action class definition.""" + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + workspace = os.path.realpath(values) + setattr(namespace, self.dest, workspace) + + +class PortAction(argparse.Action): + """Port action class definition.""" + + MIN_PORT = 1 + MAX_PORT = 65535 + + OPEN_PORT_LIMIT = 1024 + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + port = values + if not self.MIN_PORT <= port <= self.MAX_PORT: + parser.error(f'{option_string} should be chosen from {self.MIN_PORT} to {self.MAX_PORT}') + + setattr(namespace, self.dest, port) + + +class Command(BaseCommand): + """ + Start mindinsight service. + """ + + name = 'start' + description = 'startup mindinsight service' + + def add_arguments(self, parser): + """ + Add arguments to parser. + + Args: + parser (ArgumentParser): Specify parser to which arguments are added. + """ + parser.add_argument( + '--config', + type=str, + action=ConfigAction, + help=""" + Specify path for user config module or file of the form python:path.to.config.module + or file:/path/to/config.py + """) + + parser.add_argument( + '--workspace', + type=str, + action=WorkspaceAction, + help=""" + Specify path for user workspace. Default is %s. + """ % settings.WORKSPACE) + + parser.add_argument( + '--port', + type=int, + action=PortAction, + help=""" + Custom port ranging from %s to %s. Default value is %s. + """ % (PortAction.MIN_PORT, PortAction.MAX_PORT, settings.PORT)) + + for hook in HookUtils.instance().hooks(): + hook.register_startup_arguments(parser) + + def update_settings(self, args): + """ + Update settings. + + Args: + args (Namespace): parsed arguments to hold customized parameters. + """ + kwargs = {} + for key, value in args.__dict__.items(): + if value is not None: + kwargs[key] = value + + init(**kwargs) + + def run(self, args): + """ + Execute for start command. + + Args: + args (Namespace): Parsed arguments to hold customized parameters. + """ + for key, value in args.__dict__.items(): + if value is not None: + self.logger.info('%s = %s', key, value) + + try: + self.check_port() + except PortNotAvailableError as error: + print(error.message) + self.logger.error(error.message) + return + + run_module = import_module('mindinsight.backend.run') + run_module.start() + + self.logger.info('Start mindinsight done.') + + def check_port(self): + """Check port.""" + if os.getuid() != 0 and settings.PORT < PortAction.OPEN_PORT_LIMIT: + raise PortNotAvailableError( + f'Port {settings.PORT} below {PortAction.OPEN_PORT_LIMIT} is not allowed by current user.') + + connections = psutil.net_connections() + for connection in connections: + if connection.status != 'LISTEN': + continue + if connection.laddr.port == settings.PORT: + raise PortNotAvailableError(f'Port {settings.PORT} is not available for MindInsight') diff --git a/mindinsight/scripts/stop.py b/mindinsight/scripts/stop.py new file mode 100644 index 00000000..90c3832e --- /dev/null +++ b/mindinsight/scripts/stop.py @@ -0,0 +1,165 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Stop mindinsight service.""" + +import os +import argparse +import signal +import getpass + +import psutil + +from mindinsight.conf import settings +from mindinsight.utils.command import BaseCommand +from mindinsight.utils.hook import HookUtils + + +class PortAction(argparse.Action): + """Port action class definition.""" + + MIN_PORT = 1 + MAX_PORT = 65535 + + def __call__(self, parser, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + port = values + if not self.MIN_PORT <= port <= self.MAX_PORT: + parser.error(f'{option_string} should be chosen from {self.MIN_PORT} to {self.MAX_PORT}') + + setattr(namespace, self.dest, port) + + +class Command(BaseCommand): + """Stop command.""" + name = 'stop' + description = 'stop mindinsight service' + + cmd_regex = 'mindinsight.backend.application:APP' + access_log_path = os.path.join('gunicorn', 'access.log') + + def add_arguments(self, parser): + """ + Add arguments to parser. + + Args: + parser (ArgumentParser): Specify parser to which arguments are added. + """ + parser.add_argument( + '--port', + type=int, + action=PortAction, + help=""" + Custom port ranging from %s to %s. Default value is %s + """ % (PortAction.MIN_PORT, PortAction.MAX_PORT, settings.PORT)) + + def update_settings(self, args): + """ + Update settings. + + Args: + args (Namespace): parsed arguments to hold customized parameters. + """ + if args.port is None: + args.port = settings.PORT + + pid, workspace = self.get_process(args.port) + settings.config_workspace(workspace) + setattr(args, 'pid', pid) + + def run(self, args): + """ + Run to stop. + + Args: + args (Namespace): Parsed arguments to hold customized parameters. + """ + port, pid = args.port, args.pid + if not pid: + msg = f'No mindinsight service found for port {port}' + print(msg) + return + + self.logger.info('Stop mindinsight with port %s and pid %s.', port, pid) + + process = psutil.Process(pid) + child_pids = [child.pid for child in process.children()] + + # kill gunicorn master process + try: + os.kill(pid, signal.SIGKILL) + except PermissionError: + print('kill pid %s failed due to permission error' % pid) + return + + # cleanup gunicorn worker processes + for child_pid in child_pids: + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + for hook in HookUtils.instance().hooks(): + hook.on_shutdown(self.logger) + + print('Stop mindinsight service successfully') + + def get_process(self, port): + """ + Get mindinsight process + + Args: + port (int): Specified port for mindinsight process. + + Returns: + tuple, return mindinsight process pid and workspace. + """ + pid, workspace = 0, settings.WORKSPACE + user = getpass.getuser() + connections = psutil.net_connections() + for connection in connections: + if connection.status != 'LISTEN': + continue + if connection.laddr.port != port: + continue + + try: + process = psutil.Process(connection.pid) + except psutil.NoSuchProcess: + continue + + cmds = process.cmdline() + if ' '.join(cmds).find(self.cmd_regex) == -1: + continue + if user != process.username(): + continue + + pid = process.pid if process.ppid() == 1 else process.ppid() + + for open_file in process.open_files(): + if open_file.path.endswith(self.access_log_path): + log_base_dir = open_file.path[:-len(self.access_log_path)] + workspace = os.path.realpath(os.path.join(log_base_dir, os.pardir)) + break + break + + return pid, workspace diff --git a/mindinsight/ui/.browserslistrc b/mindinsight/ui/.browserslistrc new file mode 100644 index 00000000..9dee6464 --- /dev/null +++ b/mindinsight/ui/.browserslistrc @@ -0,0 +1,3 @@ +> 1% +last 2 versions +not ie <= 8 diff --git a/mindinsight/ui/.eslintrc.js b/mindinsight/ui/.eslintrc.js new file mode 100644 index 00000000..5bbfb1ee --- /dev/null +++ b/mindinsight/ui/.eslintrc.js @@ -0,0 +1,30 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module.exports = { + root: true, + env: { + node: true, + es6: true, + browser: true, + }, + extends: ['plugin:vue/essential', 'google'], + rules: { + 'max-len': [2, 120, 4, { ignoreUrls: true}], + }, + parserOptions: { + parser: 'babel-eslint', + }, +}; diff --git a/mindinsight/ui/.gitignore b/mindinsight/ui/.gitignore new file mode 100644 index 00000000..185e6631 --- /dev/null +++ b/mindinsight/ui/.gitignore @@ -0,0 +1,21 @@ +.DS_Store +node_modules +/dist + +# local env files +.env.local +.env.*.local + +# Log files +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Editor directories and files +.idea +.vscode +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw* diff --git a/mindinsight/ui/babel.config.js b/mindinsight/ui/babel.config.js new file mode 100644 index 00000000..02c5eb53 --- /dev/null +++ b/mindinsight/ui/babel.config.js @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module.exports = { + presets: [ + [ + '@vue/app', + { + polyfills: ['es6.promise', 'es6.symbol'], + }, + ], + ], +}; diff --git a/mindinsight/ui/package.json b/mindinsight/ui/package.json new file mode 100644 index 00000000..95c14a1c --- /dev/null +++ b/mindinsight/ui/package.json @@ -0,0 +1,37 @@ +{ + "name": "MindInsight", + "version": "1.0.0", + "private": true, + "scripts": { + "serve": "vue-cli-service serve", + "build": "vue-cli-service build", + "lint": "eslint src/**/*.js src/**/*.vue", + "i18n:report": "vue-cli-service i18n:report --src './src/**/*.?(js|vue)' --locales './src/locales/**/*.json'" + }, + "dependencies": { + "axios": "0.18.1", + "core-js": "3.3.2", + "d3": "5.6.0", + "d3-graphviz": "2.6.1", + "element-ui": "2.11.1", + "vue": "2.6.11", + "vue-i18n": "8.9.0", + "vue-router": "3.0.6", + "vuex": "3.1.1", + "echarts": "4.4.0" + }, + "devDependencies": { + "@intlify/vue-i18n-loader": "0.6.1", + "@vue/cli-service": "4.1.0", + "@vue/cli-plugin-babel": "4.1.0", + "babel-core": "6.26.0", + "babel-eslint": "10.0.3", + "eslint": "6.6.0", + "eslint-config-google": "0.13.0", + "eslint-plugin-vue": "5.2.3", + "sass": "1.25.0", + "sass-loader": "8.0.0", + "vue-cli-plugin-i18n": "0.6.1", + "vue-template-compiler": "2.6.11" + } +} diff --git a/mindinsight/ui/postcss.config.js b/mindinsight/ui/postcss.config.js new file mode 100644 index 00000000..9d1cb441 --- /dev/null +++ b/mindinsight/ui/postcss.config.js @@ -0,0 +1,20 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +module.exports = { + plugins: { + autoprefixer: {}, + }, +}; diff --git a/mindinsight/ui/public/index.html b/mindinsight/ui/public/index.html new file mode 100644 index 00000000..d1bd5713 --- /dev/null +++ b/mindinsight/ui/public/index.html @@ -0,0 +1,39 @@ + + + + + + + + + + + MindInsight + + + + +
+ + + diff --git a/mindinsight/ui/public/static/img/favicon.ico b/mindinsight/ui/public/static/img/favicon.ico new file mode 100644 index 00000000..503bf9c2 Binary files /dev/null and b/mindinsight/ui/public/static/img/favicon.ico differ diff --git a/mindinsight/ui/src/app.vue b/mindinsight/ui/src/app.vue new file mode 100644 index 00000000..678df636 --- /dev/null +++ b/mindinsight/ui/src/app.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/mindinsight/ui/src/assets/css/element.css b/mindinsight/ui/src/assets/css/element.css new file mode 100644 index 00000000..e440e734 --- /dev/null +++ b/mindinsight/ui/src/assets/css/element.css @@ -0,0 +1,15962 @@ +@charset "UTF-8"; + +.el-pagination--small .arrow.disabled, +.el-table .hidden-columns, +.el-table td.is-hidden > *, +.el-table th.is-hidden > *, +.el-table--hidden { + visibility: hidden; +} + +.el-dropdown .el-dropdown-selfdefine:focus:active, +.el-dropdown .el-dropdown-selfdefine:focus:not(.focusing), +.el-message__closeBtn:focus, +.el-message__content:focus, +.el-popover:focus, +.el-popover:focus:active, +.el-popover__reference:focus:hover, +.el-popover__reference:focus:not(.focusing), +.el-rate:active, +.el-rate:focus, +.el-tooltip:focus:hover, +.el-tooltip:focus:not(.focusing), +.el-upload-list__item.is-success:active, +.el-upload-list__item.is-success:not(.focusing):focus { + outline-width: 0; +} + +.el-input__suffix, +.el-tree.is-dragging .el-tree-node__content * { + pointer-events: none; +} + +@font-face { + font-family: element-icons; + src: url(../fonts/element-icons.woff) format('woff'), + url(../fonts/element-icons.ttf) format('truetype'); + font-weight: 400; + font-display: 'auto'; + font-style: normal; +} + +[class*=' el-icon-'], +[class^='el-icon-'] { + font-family: element-icons !important; + speak: none; + font-style: normal; + font-weight: 400; + font-variant: normal; + text-transform: none; + line-height: 1; + vertical-align: baseline; + display: inline-block; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.el-icon-ice-cream-round:before { + content: '\e6a0'; +} + +.el-icon-ice-cream-square:before { + content: '\e6a3'; +} + +.el-icon-lollipop:before { + content: '\e6a4'; +} + +.el-icon-potato-strips:before { + content: '\e6a5'; +} + +.el-icon-milk-tea:before { + content: '\e6a6'; +} + +.el-icon-ice-drink:before { + content: '\e6a7'; +} + +.el-icon-ice-tea:before { + content: '\e6a9'; +} + +.el-icon-coffee:before { + content: '\e6aa'; +} + +.el-icon-orange:before { + content: '\e6ab'; +} + +.el-icon-pear:before { + content: '\e6ac'; +} + +.el-icon-apple:before { + content: '\e6ad'; +} + +.el-icon-cherry:before { + content: '\e6ae'; +} + +.el-icon-watermelon:before { + content: '\e6af'; +} + +.el-icon-grape:before { + content: '\e6b0'; +} + +.el-icon-refrigerator:before { + content: '\e6b1'; +} + +.el-icon-goblet-square-full:before { + content: '\e6b2'; +} + +.el-icon-goblet-square:before { + content: '\e6b3'; +} + +.el-icon-goblet-full:before { + content: '\e6b4'; +} + +.el-icon-goblet:before { + content: '\e6b5'; +} + +.el-icon-cold-drink:before { + content: '\e6b6'; +} + +.el-icon-coffee-cup:before { + content: '\e6b8'; +} + +.el-icon-water-cup:before { + content: '\e6b9'; +} + +.el-icon-hot-water:before { + content: '\e6ba'; +} + +.el-icon-ice-cream:before { + content: '\e6bb'; +} + +.el-icon-dessert:before { + content: '\e6bc'; +} + +.el-icon-sugar:before { + content: '\e6bd'; +} + +.el-icon-tableware:before { + content: '\e6be'; +} + +.el-icon-burger:before { + content: '\e6bf'; +} + +.el-icon-knife-fork:before { + content: '\e6c1'; +} + +.el-icon-fork-spoon:before { + content: '\e6c2'; +} + +.el-icon-chicken:before { + content: '\e6c3'; +} + +.el-icon-food:before { + content: '\e6c4'; +} + +.el-icon-dish-1:before { + content: '\e6c5'; +} + +.el-icon-dish:before { + content: '\e6c6'; +} + +.el-icon-moon-night:before { + content: '\e6ee'; +} + +.el-icon-moon:before { + content: '\e6f0'; +} + +.el-icon-cloudy-and-sunny:before { + content: '\e6f1'; +} + +.el-icon-partly-cloudy:before { + content: '\e6f2'; +} + +.el-icon-cloudy:before { + content: '\e6f3'; +} + +.el-icon-sunny:before { + content: '\e6f6'; +} + +.el-icon-sunset:before { + content: '\e6f7'; +} + +.el-icon-sunrise-1:before { + content: '\e6f8'; +} + +.el-icon-sunrise:before { + content: '\e6f9'; +} + +.el-icon-heavy-rain:before { + content: '\e6fa'; +} + +.el-icon-lightning:before { + content: '\e6fb'; +} + +.el-icon-light-rain:before { + content: '\e6fc'; +} + +.el-icon-wind-power:before { + content: '\e6fd'; +} + +.el-icon-baseball:before { + content: '\e712'; +} + +.el-icon-soccer:before { + content: '\e713'; +} + +.el-icon-football:before { + content: '\e715'; +} + +.el-icon-basketball:before { + content: '\e716'; +} + +.el-icon-ship:before { + content: '\e73f'; +} + +.el-icon-truck:before { + content: '\e740'; +} + +.el-icon-bicycle:before { + content: '\e741'; +} + +.el-icon-mobile-phone:before { + content: '\e6d3'; +} + +.el-icon-service:before { + content: '\e6d4'; +} + +.el-icon-key:before { + content: '\e6e2'; +} + +.el-icon-unlock:before { + content: '\e6e4'; +} + +.el-icon-lock:before { + content: '\e6e5'; +} + +.el-icon-watch:before { + content: '\e6fe'; +} + +.el-icon-watch-1:before { + content: '\e6ff'; +} + +.el-icon-timer:before { + content: '\e702'; +} + +.el-icon-alarm-clock:before { + content: '\e703'; +} + +.el-icon-map-location:before { + content: '\e704'; +} + +.el-icon-delete-location:before { + content: '\e705'; +} + +.el-icon-add-location:before { + content: '\e706'; +} + +.el-icon-location-information:before { + content: '\e707'; +} + +.el-icon-location-outline:before { + content: '\e708'; +} + +.el-icon-location:before { + content: '\e79e'; +} + +.el-icon-place:before { + content: '\e709'; +} + +.el-icon-discover:before { + content: '\e70a'; +} + +.el-icon-first-aid-kit:before { + content: '\e70b'; +} + +.el-icon-trophy-1:before { + content: '\e70c'; +} + +.el-icon-trophy:before { + content: '\e70d'; +} + +.el-icon-medal:before { + content: '\e70e'; +} + +.el-icon-medal-1:before { + content: '\e70f'; +} + +.el-icon-stopwatch:before { + content: '\e710'; +} + +.el-icon-mic:before { + content: '\e711'; +} + +.el-icon-copy-document:before { + content: '\e718'; +} + +.el-icon-full-screen:before { + content: '\e719'; +} + +.el-icon-switch-button:before { + content: '\e71b'; +} + +.el-icon-aim:before { + content: '\e71c'; +} + +.el-icon-crop:before { + content: '\e71d'; +} + +.el-icon-odometer:before { + content: '\e71e'; +} + +.el-icon-time:before { + content: '\e71f'; +} + +.el-icon-bangzhu:before { + content: '\e724'; +} + +.el-icon-close-notification:before { + content: '\e726'; +} + +.el-icon-microphone:before { + content: '\e727'; +} + +.el-icon-turn-off-microphone:before { + content: '\e728'; +} + +.el-icon-position:before { + content: '\e729'; +} + +.el-icon-postcard:before { + content: '\e72a'; +} + +.el-icon-message:before { + content: '\e72b'; +} + +.el-icon-chat-line-square:before { + content: '\e72d'; +} + +.el-icon-chat-dot-square:before { + content: '\e72e'; +} + +.el-icon-chat-dot-round:before { + content: '\e72f'; +} + +.el-icon-chat-square:before { + content: '\e730'; +} + +.el-icon-chat-line-round:before { + content: '\e731'; +} + +.el-icon-chat-round:before { + content: '\e732'; +} + +.el-icon-set-up:before { + content: '\e733'; +} + +.el-icon-turn-off:before { + content: '\e734'; +} + +.el-icon-open:before { + content: '\e735'; +} + +.el-icon-connection:before { + content: '\e736'; +} + +.el-icon-link:before { + content: '\e737'; +} + +.el-icon-cpu:before { + content: '\e738'; +} + +.el-icon-thumb:before { + content: '\e739'; +} + +.el-icon-female:before { + content: '\e73a'; +} + +.el-icon-male:before { + content: '\e73b'; +} + +.el-icon-guide:before { + content: '\e73c'; +} + +.el-icon-news:before { + content: '\e73e'; +} + +.el-icon-price-tag:before { + content: '\e744'; +} + +.el-icon-discount:before { + content: '\e745'; +} + +.el-icon-wallet:before { + content: '\e747'; +} + +.el-icon-coin:before { + content: '\e748'; +} + +.el-icon-money:before { + content: '\e749'; +} + +.el-icon-bank-card:before { + content: '\e74a'; +} + +.el-icon-box:before { + content: '\e74b'; +} + +.el-icon-present:before { + content: '\e74c'; +} + +.el-icon-sell:before { + content: '\e6d5'; +} + +.el-icon-sold-out:before { + content: '\e6d6'; +} + +.el-icon-shopping-bag-2:before { + content: '\e74d'; +} + +.el-icon-shopping-bag-1:before { + content: '\e74e'; +} + +.el-icon-shopping-cart-2:before { + content: '\e74f'; +} + +.el-icon-shopping-cart-1:before { + content: '\e750'; +} + +.el-icon-shopping-cart-full:before { + content: '\e751'; +} + +.el-icon-smoking:before { + content: '\e752'; +} + +.el-icon-no-smoking:before { + content: '\e753'; +} + +.el-icon-house:before { + content: '\e754'; +} + +.el-icon-table-lamp:before { + content: '\e755'; +} + +.el-icon-school:before { + content: '\e756'; +} + +.el-icon-office-building:before { + content: '\e757'; +} + +.el-icon-toilet-paper:before { + content: '\e758'; +} + +.el-icon-notebook-2:before { + content: '\e759'; +} + +.el-icon-notebook-1:before { + content: '\e75a'; +} + +.el-icon-files:before { + content: '\e75b'; +} + +.el-icon-collection:before { + content: '\e75c'; +} + +.el-icon-receiving:before { + content: '\e75d'; +} + +.el-icon-suitcase-1:before { + content: '\e760'; +} + +.el-icon-suitcase:before { + content: '\e761'; +} + +.el-icon-film:before { + content: '\e763'; +} + +.el-icon-collection-tag:before { + content: '\e765'; +} + +.el-icon-data-analysis:before { + content: '\e766'; +} + +.el-icon-pie-chart:before { + content: '\e767'; +} + +.el-icon-data-board:before { + content: '\e768'; +} + +.el-icon-data-line:before { + content: '\e76d'; +} + +.el-icon-reading:before { + content: '\e769'; +} + +.el-icon-magic-stick:before { + content: '\e76a'; +} + +.el-icon-coordinate:before { + content: '\e76b'; +} + +.el-icon-mouse:before { + content: '\e76c'; +} + +.el-icon-brush:before { + content: '\e76e'; +} + +.el-icon-headset:before { + content: '\e76f'; +} + +.el-icon-umbrella:before { + content: '\e770'; +} + +.el-icon-scissors:before { + content: '\e771'; +} + +.el-icon-mobile:before { + content: '\e773'; +} + +.el-icon-attract:before { + content: '\e774'; +} + +.el-icon-monitor:before { + content: '\e775'; +} + +.el-icon-search:before { + content: '\e778'; +} + +.el-icon-takeaway-box:before { + content: '\e77a'; +} + +.el-icon-paperclip:before { + content: '\e77d'; +} + +.el-icon-printer:before { + content: '\e77e'; +} + +.el-icon-document-add:before { + content: '\e782'; +} + +.el-icon-document:before { + content: '\e785'; +} + +.el-icon-document-checked:before { + content: '\e786'; +} + +.el-icon-document-copy:before { + content: '\e787'; +} + +.el-icon-document-delete:before { + content: '\e788'; +} + +.el-icon-document-remove:before { + content: '\e789'; +} + +.el-icon-tickets:before { + content: '\e78b'; +} + +.el-icon-folder-checked:before { + content: '\e77f'; +} + +.el-icon-folder-delete:before { + content: '\e780'; +} + +.el-icon-folder-remove:before { + content: '\e781'; +} + +.el-icon-folder-add:before { + content: '\e783'; +} + +.el-icon-folder-opened:before { + content: '\e784'; +} + +.el-icon-folder:before { + content: '\e78a'; +} + +.el-icon-edit-outline:before { + content: '\e764'; +} + +.el-icon-edit:before { + content: '\e78c'; +} + +.el-icon-date:before { + content: '\e78e'; +} + +.el-icon-c-scale-to-original:before { + content: '\e7c6'; +} + +.el-icon-view:before { + content: '\e6ce'; +} + +.el-icon-loading:before { + content: '\e6cf'; +} + +.el-icon-rank:before { + content: '\e6d1'; +} + +.el-icon-sort-down:before { + content: '\e7c4'; +} + +.el-icon-sort-up:before { + content: '\e7c5'; +} + +.el-icon-sort:before { + content: '\e6d2'; +} + +.el-icon-finished:before { + content: '\e6cd'; +} + +.el-icon-refresh-left:before { + content: '\e6c7'; +} + +.el-icon-refresh-right:before { + content: '\e6c8'; +} + +.el-icon-refresh:before { + content: '\e6d0'; +} + +.el-icon-video-play:before { + content: '\e7c0'; +} + +.el-icon-video-pause:before { + content: '\e7c1'; +} + +.el-icon-d-arrow-right:before { + content: '\e6dc'; +} + +.el-icon-d-arrow-left:before { + content: '\e6dd'; +} + +.el-icon-arrow-up:before { + content: '\e6e1'; +} + +.el-icon-arrow-down:before { + content: '\e6df'; +} + +.el-icon-arrow-right:before { + content: '\e6e0'; +} + +.el-icon-arrow-left:before { + content: '\e6de'; +} + +.el-icon-top-right:before { + content: '\e6e7'; +} + +.el-icon-top-left:before { + content: '\e6e8'; +} + +.el-icon-top:before { + content: '\e6e6'; +} + +.el-icon-bottom:before { + content: '\e6eb'; +} + +.el-icon-right:before { + content: '\e6e9'; +} + +.el-icon-back:before { + content: '\e6ea'; +} + +.el-icon-bottom-right:before { + content: '\e6ec'; +} + +.el-icon-bottom-left:before { + content: '\e6ed'; +} + +.el-icon-caret-top:before { + content: '\e78f'; +} + +.el-icon-caret-bottom:before { + content: '\e790'; +} + +.el-icon-caret-right:before { + content: '\e791'; +} + +.el-icon-caret-left:before { + content: '\e792'; +} + +.el-icon-d-caret:before { + content: '\e79a'; +} + +.el-icon-share:before { + content: '\e793'; +} + +.el-icon-menu:before { + content: '\e798'; +} + +.el-icon-s-grid:before { + content: '\e7a6'; +} + +.el-icon-s-check:before { + content: '\e7a7'; +} + +.el-icon-s-data:before { + content: '\e7a8'; +} + +.el-icon-s-opportunity:before { + content: '\e7aa'; +} + +.el-icon-s-custom:before { + content: '\e7ab'; +} + +.el-icon-s-claim:before { + content: '\e7ad'; +} + +.el-icon-s-finance:before { + content: '\e7ae'; +} + +.el-icon-s-comment:before { + content: '\e7af'; +} + +.el-icon-s-flag:before { + content: '\e7b0'; +} + +.el-icon-s-marketing:before { + content: '\e7b1'; +} + +.el-icon-s-shop:before { + content: '\e7b4'; +} + +.el-icon-s-open:before { + content: '\e7b5'; +} + +.el-icon-s-management:before { + content: '\e7b6'; +} + +.el-icon-s-ticket:before { + content: '\e7b7'; +} + +.el-icon-s-release:before { + content: '\e7b8'; +} + +.el-icon-s-home:before { + content: '\e7b9'; +} + +.el-icon-s-promotion:before { + content: '\e7ba'; +} + +.el-icon-s-operation:before { + content: '\e7bb'; +} + +.el-icon-s-unfold:before { + content: '\e7bc'; +} + +.el-icon-s-fold:before { + content: '\e7a9'; +} + +.el-icon-s-platform:before { + content: '\e7bd'; +} + +.el-icon-s-order:before { + content: '\e7be'; +} + +.el-icon-s-cooperation:before { + content: '\e7bf'; +} + +.el-icon-bell:before { + content: '\e725'; +} + +.el-icon-message-solid:before { + content: '\e799'; +} + +.el-icon-video-camera:before { + content: '\e772'; +} + +.el-icon-video-camera-solid:before { + content: '\e796'; +} + +.el-icon-camera:before { + content: '\e779'; +} + +.el-icon-camera-solid:before { + content: '\e79b'; +} + +.el-icon-download:before { + content: '\e77c'; +} + +.el-icon-upload2:before { + content: '\e77b'; +} + +.el-icon-upload:before { + content: '\e7c3'; +} + +.el-icon-picture-outline-round:before { + content: '\e75f'; +} + +.el-icon-picture-outline:before { + content: '\e75e'; +} + +.el-icon-picture:before { + content: '\e79f'; +} + +.el-icon-close:before { + content: '\e6db'; +} + +.el-icon-check:before { + content: '\e6da'; +} + +.el-icon-plus:before { + content: '\e6d9'; +} + +.el-icon-minus:before { + content: '\e6d8'; +} + +.el-icon-help:before { + content: '\e73d'; +} + +.el-icon-s-help:before { + content: '\e7b3'; +} + +.el-icon-circle-close:before { + content: '\e78d'; +} + +.el-icon-circle-check:before { + content: '\e720'; +} + +.el-icon-circle-plus-outline:before { + content: '\e723'; +} + +.el-icon-remove-outline:before { + content: '\e722'; +} + +.el-icon-zoom-out:before { + content: '\e776'; +} + +.el-icon-zoom-in:before { + content: '\e777'; +} + +.el-icon-error:before { + content: '\e79d'; +} + +.el-icon-success:before { + content: '\e79c'; +} + +.el-icon-circle-plus:before { + content: '\e7a0'; +} + +.el-icon-remove:before { + content: '\e7a2'; +} + +.el-icon-info:before { + content: '\e7a1'; +} + +.el-icon-question:before { + content: '\e7a4'; +} + +.el-icon-warning-outline:before { + content: '\e6c9'; +} + +.el-icon-warning:before { + content: '\e7a3'; +} + +.el-icon-goods:before { + content: '\e7c2'; +} + +.el-icon-s-goods:before { + content: '\e7b2'; +} + +.el-icon-star-off:before { + content: '\e717'; +} + +.el-icon-star-on:before { + content: '\e797'; +} + +.el-icon-more-outline:before { + content: '\e6cc'; +} + +.el-icon-more:before { + content: '\e794'; +} + +.el-icon-phone-outline:before { + content: '\e6cb'; +} + +.el-icon-phone:before { + content: '\e795'; +} + +.el-icon-user:before { + content: '\e6e3'; +} + +.el-icon-user-solid:before { + content: '\e7a5'; +} + +.el-icon-setting:before { + content: '\e6ca'; +} + +.el-icon-s-tools:before { + content: '\e7ac'; +} + +.el-icon-delete:before { + content: '\e6d7'; +} + +.el-icon-delete-solid:before { + content: '\e7c9'; +} + +.el-icon-eleme:before { + content: '\e7c7'; +} + +.el-icon-platform-eleme:before { + content: '\e7ca'; +} + +.el-icon-loading { + -webkit-animation: rotating 2s linear infinite; + animation: rotating 2s linear infinite; +} + +.el-icon--right { + margin-left: 5px; +} + +.el-icon--left { + margin-right: 5px; +} + +@-webkit-keyframes rotating { + 0% { + -webkit-transform: rotateZ(0); + transform: rotateZ(0); + } + + 100% { + -webkit-transform: rotateZ(360deg); + transform: rotateZ(360deg); + } +} + +@keyframes rotating { + 0% { + -webkit-transform: rotateZ(0); + transform: rotateZ(0); + } + + 100% { + -webkit-transform: rotateZ(360deg); + transform: rotateZ(360deg); + } +} + +.el-pagination { + white-space: nowrap; + padding: 2px 5px; + color: #303133; + font-weight: 700; +} + +.el-pagination::after, +.el-pagination::before { + display: table; + content: ''; +} + +.el-pagination::after { + clear: both; +} + +.el-pagination button, +.el-pagination span:not([class*='suffix']) { + display: inline-block; + font-size: 13px; + min-width: 35.5px; + height: 28px; + line-height: 28px; + vertical-align: top; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-pagination .el-input__inner { + text-align: center; + -moz-appearance: textfield; + line-height: normal; +} + +.el-pagination .el-input__suffix { + right: 0; + -webkit-transform: scale(0.8); + transform: scale(0.8); +} + +.el-pagination .el-select .el-input { + width: 100px; + margin: 0 5px; +} + +.el-pagination .el-select .el-input .el-input__inner { + padding-right: 25px; + border-radius: 3px; +} + +.el-pagination button { + border: none; + padding: 0 6px; + background: 0 0; +} + +.el-pagination button:focus { + outline: 0; +} + +.el-pagination button:hover { + color: #00a5a7; +} + +.el-pagination button:disabled { + color: #c0c4cc; + background-color: #fff; + cursor: not-allowed; +} + +.el-pagination .btn-next, +.el-pagination .btn-prev { + background: center center no-repeat #e6ebf5; + background-size: 16px; + cursor: pointer; + margin: 0; + color: #303133; +} + +.el-pagination .btn-next .el-icon, +.el-pagination .btn-prev .el-icon { + display: block; + font-size: 12px; + font-weight: 700; + color: #b8becc; +} + +.el-pagination button, +.el-pagination span:not([class*='suffix']) { + min-width: 25px; +} + +.el-pagination .el-pager li.disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-pager li, +.el-pager li.btn-quicknext:hover, +.el-pager li.btn-quickprev:hover { + cursor: pointer; +} + +.el-pagination--small .btn-next, +.el-pagination--small .btn-prev, +.el-pagination--small .el-pager li, +.el-pagination--small .el-pager li.btn-quicknext, +.el-pagination--small .el-pager li.btn-quickprev, +.el-pagination--small .el-pager li:last-child { + border-color: transparent; + font-size: 12px; + line-height: 22px; + height: 22px; + min-width: 22px; +} + +.el-pagination--small .more::before, +.el-pagination--small li.more::before { + line-height: 24px; +} + +.el-pagination--small button, +.el-pagination--small span:not([class*='suffix']) { + height: 22px; + line-height: 22px; +} + +.el-pagination--small .el-pagination__editor, +.el-pagination--small .el-pagination__editor.el-input .el-input__inner { + height: 22px; +} + +.el-pagination__sizes { + margin: 0 10px 0 0; + font-weight: 400; + color: #606266; +} + +.el-pagination__sizes .el-input .el-input__inner { + font-size: 13px; + padding-left: 8px; +} + +.el-pagination__sizes .el-input .el-input__inner:hover { + border-color: #00a5a7; +} + +.el-pagination__total { + margin-right: 10px; + font-weight: 400; + color: #606266; +} + +.el-pagination__jump { + margin-left: 24px; + font-weight: 400; + color: #606266; +} + +.el-pagination__jump .el-input__inner { + padding: 0 3px; +} + +.el-pagination__rightwrapper { + float: right; +} + +.el-pagination__editor { + line-height: 18px; + padding: 0 2px; + height: 28px; + text-align: center; + margin: 0 2px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-radius: 3px; +} + +.el-pager, +.el-pagination.is-background .btn-next, +.el-pagination.is-background .btn-prev { + padding: 0; +} + +.el-pagination__editor.el-input { + width: 50px; +} + +.el-pagination__editor.el-input .el-input__inner { + height: 28px; +} + +.el-pagination__editor .el-input__inner::-webkit-inner-spin-button, +.el-pagination__editor .el-input__inner::-webkit-outer-spin-button { + -webkit-appearance: none; + margin: 0; +} + +.el-pagination.is-background .btn-next, +.el-pagination.is-background .btn-prev, +.el-pagination.is-background .el-pager li { + margin: 0 5px; + background-color: #f4f4f5; + color: #606266; + min-width: 30px; + border-radius: 2px; +} + +.el-pagination.is-background .btn-next.disabled, +.el-pagination.is-background .btn-next:disabled, +.el-pagination.is-background .btn-prev.disabled, +.el-pagination.is-background .btn-prev:disabled, +.el-pagination.is-background .el-pager li.disabled { + color: #c0c4cc; +} + +.el-pagination.is-background .el-pager li:not(.disabled):hover { + color: #00a5a7; +} + +.el-pagination.is-background .el-pager li:not(.disabled).active { + color: #00a5a7; +} + +.el-dialog, +.el-pager li { + background: #fff; + -webkit-box-sizing: border-box; +} + +.el-pagination.is-background.el-pagination--small .btn-next, +.el-pagination.is-background.el-pagination--small .btn-prev, +.el-pagination.is-background.el-pagination--small .el-pager li { + margin: 0 3px; + min-width: 22px; +} + +.el-pager, +.el-pager li { + vertical-align: top; + margin: 0; + display: inline-block; +} + +.el-pager { + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + list-style: none; + font-size: 0; +} + +.el-date-table, +.el-table th { + -webkit-user-select: none; + -ms-user-select: none; +} + +.el-pager .more::before { + line-height: 30px; +} + +.el-pager li { + padding: 0 4px; + font-size: 13px; + min-width: 35.5px; + height: 28px; + line-height: 28px; + box-sizing: border-box; + text-align: center; +} + +.el-menu--collapse .el-menu .el-submenu, +.el-menu--popup { + min-width: 200px; +} + +.el-pager li.btn-quicknext, +.el-pager li.btn-quickprev { + line-height: 28px; + color: #303133; +} + +.el-pager li.btn-quicknext.disabled, +.el-pager li.btn-quickprev.disabled { + color: #c0c4cc; +} + +.el-pager li.active + li { + border-left: 0; +} + +.el-pager li:hover { + color: #00a5a7; +} + +.el-pager li.active { + color: #00a5a7; + cursor: default; +} + +@-webkit-keyframes v-modal-in { + 0% { + opacity: 0; + } +} + +@-webkit-keyframes v-modal-out { + 100% { + opacity: 0; + } +} + +.el-dialog { + position: relative; + margin: 0 auto 50px; + border-radius: 2px; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3); + box-sizing: border-box; + width: 50%; +} + +.el-dialog.is-fullscreen { + width: 100%; + margin-top: 0; + margin-bottom: 0; + height: 100%; + overflow: auto; +} + +.el-dialog__wrapper { + position: fixed; + top: 0; + right: 0; + bottom: 0; + left: 0; + overflow: auto; + margin: 0; +} + +.el-dialog__header { + padding: 20px 20px 10px; +} + +.el-dialog__headerbtn { + position: absolute; + top: 20px; + right: 20px; + padding: 0; + background: 0 0; + border: none; + outline: 0; + cursor: pointer; + font-size: 16px; +} + +.el-dialog__headerbtn .el-dialog__close { + color: #909399; +} + +.el-dialog__headerbtn:focus .el-dialog__close, +.el-dialog__headerbtn:hover .el-dialog__close { + color: #00a5a7; +} + +.el-dialog__title { + line-height: 24px; + font-size: 18px; + color: #303133; +} + +.el-dialog__body { + padding: 30px 20px; + color: #606266; + font-size: 14px; + word-break: break-all; +} + +.el-dialog__footer { + padding: 10px 20px 20px; + text-align: right; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-dialog--center { + text-align: center; +} + +.el-dialog--center .el-dialog__body { + text-align: initial; + padding: 25px 25px 30px; +} + +.el-dialog--center .el-dialog__footer { + text-align: inherit; +} + +.dialog-fade-enter-active { + -webkit-animation: dialog-fade-in 0.3s; + animation: dialog-fade-in 0.3s; +} + +.dialog-fade-leave-active { + -webkit-animation: dialog-fade-out 0.3s; + animation: dialog-fade-out 0.3s; +} + +@-webkit-keyframes dialog-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@keyframes dialog-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@-webkit-keyframes dialog-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +@keyframes dialog-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +.el-autocomplete { + position: relative; + display: inline-block; +} + +.el-autocomplete-suggestion { + margin: 5px 0; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + border-radius: 4px; + border: 1px solid #e4e7ed; + -webkit-box-sizing: border-box; + box-sizing: border-box; + background-color: #fff; +} + +.el-dropdown-menu, +.el-menu--collapse .el-submenu .el-menu { + z-index: 10; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-autocomplete-suggestion__wrap { + max-height: 280px; + padding: 10px 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-autocomplete-suggestion__list { + margin: 0; + padding: 0; +} + +.el-autocomplete-suggestion li { + padding: 0 20px; + margin: 0; + line-height: 34px; + cursor: pointer; + color: #606266; + font-size: 14px; + list-style: none; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.el-autocomplete-suggestion li.highlighted, +.el-autocomplete-suggestion li:hover { + background-color: #f5f7fa; +} + +.el-autocomplete-suggestion li.divider { + margin-top: 6px; + border-top: 1px solid #000; +} + +.el-autocomplete-suggestion li.divider:last-child { + margin-bottom: -6px; +} + +.el-autocomplete-suggestion.is-loading li { + text-align: center; + height: 100px; + line-height: 100px; + font-size: 20px; + color: #999; +} + +.el-autocomplete-suggestion.is-loading li::after { + display: inline-block; + content: ''; + height: 100%; + vertical-align: middle; +} + +.el-autocomplete-suggestion.is-loading li:hover { + background-color: #fff; +} + +.el-autocomplete-suggestion.is-loading .el-icon-loading { + vertical-align: middle; +} + +.el-dropdown { + display: inline-block; + position: relative; + color: #606266; + font-size: 14px; +} + +.el-dropdown .el-button-group { + display: block; +} + +.el-dropdown .el-button-group .el-button { + float: none; +} + +.el-dropdown .el-dropdown__caret-button { + padding-left: 5px; + padding-right: 5px; + position: relative; + border-left: none; +} + +.el-dropdown .el-dropdown__caret-button::before { + content: ''; + position: absolute; + display: block; + width: 1px; + top: 5px; + bottom: 5px; + left: 0; + background: rgba(255, 255, 255, 0.5); +} + +.el-dropdown .el-dropdown__caret-button.el-button--default::before { + background: rgba(220, 223, 230, 0.5); +} + +.el-dropdown .el-dropdown__caret-button:hover::before { + top: 0; + bottom: 0; +} + +.el-dropdown .el-dropdown__caret-button .el-dropdown__icon { + padding-left: 0; +} + +.el-dropdown__icon { + font-size: 12px; + margin: 0 3px; +} + +.el-dropdown-menu { + position: absolute; + top: 0; + left: 0; + padding: 10px 0; + margin: 5px 0; + background-color: #fff; + border: 1px solid #ebeef5; + border-radius: 4px; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-dropdown-menu__item { + list-style: none; + line-height: 36px; + padding: 0 20px; + margin: 0; + font-size: 14px; + color: #606266; + cursor: pointer; + outline: 0; +} + +.el-dropdown-menu__item:focus, +.el-dropdown-menu__item:not(.is-disabled):hover { + background-color: rgb(230, 246, 246); + color: rgb(51, 183, 185); +} + +.el-dropdown-menu__item i { + margin-right: 5px; +} + +.el-dropdown-menu__item--divided { + position: relative; + margin-top: 6px; + border-top: 1px solid #ebeef5; +} + +.el-dropdown-menu__item--divided:before { + content: ''; + height: 6px; + display: block; + margin: 0 -20px; + background-color: #fff; +} + +.el-dropdown-menu__item.is-disabled { + cursor: default; + color: #bbb; + pointer-events: none; +} + +.el-dropdown-menu--medium { + padding: 6px 0; +} + +.el-dropdown-menu--medium .el-dropdown-menu__item { + line-height: 30px; + padding: 0 17px; + font-size: 14px; +} + +.el-dropdown-menu--medium .el-dropdown-menu__item.el-dropdown-menu__item--divided { + margin-top: 6px; +} + +.el-dropdown-menu--medium .el-dropdown-menu__item.el-dropdown-menu__item--divided:before { + height: 6px; + margin: 0 -17px; +} + +.el-dropdown-menu--small { + padding: 6px 0; +} + +.el-dropdown-menu--small .el-dropdown-menu__item { + line-height: 27px; + padding: 0 15px; + font-size: 13px; +} + +.el-dropdown-menu--small .el-dropdown-menu__item.el-dropdown-menu__item--divided { + margin-top: 4px; +} + +.el-dropdown-menu--small .el-dropdown-menu__item.el-dropdown-menu__item--divided:before { + height: 4px; + margin: 0 -15px; +} + +.el-dropdown-menu--mini { + padding: 3px 0; +} + +.el-dropdown-menu--mini .el-dropdown-menu__item { + line-height: 24px; + padding: 0 10px; + font-size: 12px; +} + +.el-dropdown-menu--mini .el-dropdown-menu__item.el-dropdown-menu__item--divided { + margin-top: 3px; +} + +.el-dropdown-menu--mini .el-dropdown-menu__item.el-dropdown-menu__item--divided:before { + height: 3px; + margin: 0 -10px; +} + +.el-menu { + border-right: solid 1px #192239; + list-style: none; + position: relative; + margin: 0; + padding-left: 0; +} + +.el-menu--horizontal > .el-menu-item:not(.is-disabled):focus, +.el-menu--horizontal > .el-menu-item:not(.is-disabled):hover, +.el-menu--horizontal > .el-submenu .el-submenu__title:hover { + /* background-color: #fff */ +} + +.el-menu::after, +.el-menu::before { + display: table; + content: ''; +} + +.el-menu::after { + clear: both; +} + +.el-menu.el-menu--horizontal { + border-bottom: solid 1px #e6e6e6; +} + +.el-menu--horizontal { + border-right: none; +} + +.el-menu--horizontal > .el-menu-item { + float: left; + height: 60px; + line-height: 60px; + margin: 0; + border-bottom: 2px solid transparent; + color: #909399; +} + +.el-menu--horizontal > .el-menu-item a, +.el-menu--horizontal > .el-menu-item a:hover { + color: inherit; +} + +.el-menu--horizontal > .el-submenu { + float: left; +} + +.el-menu--horizontal > .el-submenu:focus, +.el-menu--horizontal > .el-submenu:hover { + outline: 0; +} + +.el-menu--horizontal > .el-submenu:focus .el-submenu__title, +.el-menu--horizontal > .el-submenu:hover .el-submenu__title { + color: #fff; +} + +.el-menu--horizontal > .el-submenu.is-active .el-submenu__title { + border-bottom: 2px solid #00a5a7; + color: #fff; +} + +.el-menu--horizontal > .el-submenu .el-submenu__title { + height: 60px; + line-height: 60px; + border-bottom: 2px solid transparent; + color: #909399; +} + +.el-menu--horizontal > .el-submenu .el-submenu__icon-arrow { + position: static; + vertical-align: middle; + margin-left: 8px; + margin-top: -3px; +} + +.el-menu--horizontal .el-menu .el-menu-item, +.el-menu--horizontal .el-menu .el-submenu__title { + background-color: #fff; + float: none; + height: 36px; + line-height: 36px; + padding: 0 10px; + color: #909399; +} + +.el-menu--horizontal .el-menu .el-menu-item.is-active, +.el-menu--horizontal .el-menu .el-submenu.is-active > .el-submenu__title { + color: #303133; +} + +.el-menu--horizontal .el-menu-item:not(.is-disabled):focus, +.el-menu--horizontal .el-menu-item:not(.is-disabled):hover { + outline: 0; + color: #303133; +} + +.el-menu--horizontal > .el-menu-item.is-active { + border-bottom: 2px solid #00a5a7; + color: #303133; +} + +.el-menu--collapse { + width: 64px; +} + +.el-menu--collapse > .el-menu-item [class^='el-icon-'], +.el-menu--collapse > .el-submenu > .el-submenu__title [class^='el-icon-'] { + margin: 0; + vertical-align: middle; + width: 24px; + text-align: center; +} + +.el-menu--collapse > .el-menu-item .el-submenu__icon-arrow, +.el-menu--collapse > .el-submenu > .el-submenu__title .el-submenu__icon-arrow { + display: none; +} + +.el-menu--collapse > .el-menu-item span, +.el-menu--collapse > .el-submenu > .el-submenu__title span { + height: 0; + width: 0; + overflow: hidden; + visibility: hidden; + display: inline-block; + font-size: 20px; +} + +.el-menu--collapse > .el-menu-item.is-active i { + color: inherit; +} + +.el-menu--collapse .el-submenu { + position: relative; +} + +.el-menu--collapse .el-submenu .el-menu { + position: absolute; + margin-left: 5px; + top: 0; + left: 100%; + border: 1px solid #e4e7ed; + border-radius: 2px; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-menu-item, +.el-submenu__title { + height: 56px; + line-height: 56px; + position: relative; + box-sizing: border-box; + white-space: nowrap; + list-style: none; +} + +.el-menu--collapse .el-submenu.is-opened > .el-submenu__title .el-submenu__icon-arrow { + -webkit-transform: none; + transform: none; +} + +.el-menu--popup { + z-index: 100; + border: none; + padding: 5px 0; + border-radius: 2px; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-menu--popup-bottom-start { + margin-top: 5px; +} + +.el-menu--popup-right-start { + margin-left: 5px; + margin-right: 5px; +} + +.el-menu-item { + font-size: 14px; + color: #9ea4b3; + padding: 0 20px; + cursor: pointer; + -webkit-transition: border-color 0.3s, background-color 0.3s, color 0.3s; + transition: border-color 0.3s, background-color 0.3s, color 0.3s; + box-sizing: border-box; +} + +.el-menu-item * { + vertical-align: middle; +} + +.el-menu-item i { + color: #909399; +} + +.el-menu-item:focus, +.el-menu-item:hover { + outline: 0; + background-color: rgb(230, 246, 246); +} + +.el-menu-item.is-disabled { + opacity: 0.25; + cursor: not-allowed; + background: 0 0 !important; +} + +.el-menu-item [class^='el-icon-'] { + margin-right: 5px; + width: 24px; + text-align: center; + font-size: 18px; + vertical-align: middle; +} + +.el-menu-item.is-active { + color: #00a5a7; +} + +.el-menu-item.is-active i { + color: inherit; +} + +.el-submenu { + list-style: none; + margin: 0; + padding-left: 0; +} + +.el-submenu__title { + font-size: 14px; + color: #9ea4b3; + padding: 0 20px; + cursor: pointer; + -webkit-transition: border-color 0.3s, background-color 0.3s, color 0.3s; + transition: border-color 0.3s, background-color 0.3s, color 0.3s; + box-sizing: border-box; +} + +.el-submenu__title * { + vertical-align: middle; + font-size: 14px; +} + +.el-submenu__title i { + color: #909399; +} + +.el-submenu__title:focus, +.el-submenu__title:hover { + outline: 0; + background-color: rgb(230, 246, 246); +} + +.el-submenu__title.is-disabled { + opacity: 0.25; + cursor: not-allowed; + background: 0 0 !important; +} + +.el-submenu__title:hover { + background: none; +} + +.el-submenu .el-menu { + border: none; +} + +.el-submenu .el-menu-item { + height: 50px; + line-height: 50px; + padding: 0 45px; + min-width: 200px; +} + +.el-submenu__icon-arrow { + position: absolute; + top: 50%; + right: 20px; + margin-top: -7px; + -webkit-transition: -webkit-transform 0.3s; + transition: -webkit-transform 0.3s; + transition: transform 0.3s; + transition: transform 0.3s, -webkit-transform 0.3s; + font-size: 12px; +} + +.el-submenu.is-active .el-submenu__title { + border-bottom-color: #00a5a7; +} + +.el-submenu.is-opened > .el-submenu__title .el-submenu__icon-arrow { + -webkit-transform: rotateZ(180deg); + transform: rotateZ(180deg); +} + +.el-submenu.is-disabled .el-menu-item, +.el-submenu.is-disabled .el-submenu__title { + opacity: 0.25; + cursor: not-allowed; + background: 0 0 !important; +} + +.el-submenu [class^='el-icon-'] { + vertical-align: middle; + margin-right: 5px; + width: 24px; + text-align: center; + font-size: 18px; +} + +.el-menu-item-group > ul { + padding: 0; +} + +.el-menu-item-group__title { + padding: 7px 0 7px 20px; + line-height: normal; + font-size: 12px; + color: #909399; +} + +.el-radio-button__inner, +.el-radio-group { + display: inline-block; + line-height: 1; + vertical-align: middle; +} + +.horizontal-collapse-transition .el-submenu__title .el-submenu__icon-arrow { + -webkit-transition: 0.2s; + transition: 0.2s; + opacity: 0; +} + +.el-radio-group { + font-size: 0; +} + +.el-radio-button { + position: relative; + display: inline-block; + outline: 0; +} + +.el-radio-button__inner { + white-space: nowrap; + background: #fff; + border: 1px solid #dcdfe6; + font-weight: 500; + border-left: 0; + color: #606266; + -webkit-appearance: none; + text-align: center; + -webkit-box-sizing: border-box; + box-sizing: border-box; + outline: 0; + margin: 0; + position: relative; + cursor: pointer; + -webkit-transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + padding: 12px 20px; + font-size: 14px; + border-radius: 0; +} + +.el-radio-button__inner.is-round { + padding: 12px 20px; +} + +.el-radio-button__inner:hover { + color: #00a5a7; +} + +.el-radio-button__inner [class*='el-icon-'] { + line-height: 0.9; +} + +.el-radio-button__inner [class*='el-icon-'] + span { + margin-left: 5px; +} + +.el-radio-button:first-child .el-radio-button__inner { + border-left: 1px solid #dcdfe6; + border-radius: 4px 0 0 4px; + -webkit-box-shadow: none !important; + box-shadow: none !important; +} + +.el-radio-button__orig-radio { + opacity: 0; + outline: 0; + position: absolute; + z-index: -1; +} + +.el-radio-button__orig-radio:checked + .el-radio-button__inner { + color: #fff; + background-color: #00a5a7; + border-color: #00a5a7; + -webkit-box-shadow: -1px 0 0 0 #00a5a7; + box-shadow: -1px 0 0 0 #00a5a7; +} + +.el-radio-button__orig-radio:disabled + .el-radio-button__inner { + color: #c0c4cc; + cursor: not-allowed; + background-image: none; + background-color: #fff; + border-color: #ebeef5; + -webkit-box-shadow: none; + box-shadow: none; +} + +.el-radio-button__orig-radio:disabled:checked + .el-radio-button__inner { + background-color: #f2f6fc; +} + +.el-radio-button:last-child .el-radio-button__inner { + border-radius: 0 4px 4px 0; +} + +.el-popover, +.el-radio-button:first-child:last-child .el-radio-button__inner { + border-radius: 4px; +} + +.el-radio-button--medium .el-radio-button__inner { + padding: 10px 20px; + font-size: 14px; + border-radius: 0; +} + +.el-radio-button--medium .el-radio-button__inner.is-round { + padding: 10px 20px; +} + +.el-radio-button--small .el-radio-button__inner { + padding: 9px 15px; + font-size: 12px; + border-radius: 0; +} + +.el-radio-button--small .el-radio-button__inner.is-round { + padding: 9px 15px; +} + +.el-radio-button--mini .el-radio-button__inner { + padding: 7px 15px; + font-size: 12px; + border-radius: 0; +} + +.el-radio-button--mini .el-radio-button__inner.is-round { + padding: 7px 15px; +} + +.el-radio-button:focus:not(.is-focus):not(:active):not(.is-disabled) { + -webkit-box-shadow: 0 0 2px 2px #00a5a7; + box-shadow: 0 0 2px 2px #00a5a7; +} + +.el-switch { + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + position: relative; + font-size: 14px; + line-height: 20px; + height: 20px; + vertical-align: middle; +} + +.el-switch__core, +.el-switch__label { + display: inline-block; + cursor: pointer; +} + +.el-switch.is-disabled .el-switch__core, +.el-switch.is-disabled .el-switch__label { + cursor: not-allowed; +} + +.el-switch__label { + -webkit-transition: 0.2s; + transition: 0.2s; + height: 20px; + font-size: 14px; + font-weight: 500; + vertical-align: middle; + color: #9ea4b3; +} + +.el-switch__label.is-active { + color: #00a5a7; +} + +.el-switch__label--left { + margin-right: 10px; +} + +.el-switch__label--right { + margin-left: 10px; +} + +.el-switch__label * { + line-height: 1; + font-size: 14px; + display: inline-block; +} + +.el-switch__input { + position: absolute; + width: 0; + height: 0; + opacity: 0; + margin: 0; +} + +.el-switch__core { + margin: 0; + position: relative; + width: 40px; + height: 20px; + border: 1px solid #dcdfe6; + outline: 0; + border-radius: 10px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + background: #dcdfe6; + -webkit-transition: border-color 0.3s, background-color 0.3s; + transition: border-color 0.3s, background-color 0.3s; + vertical-align: middle; +} + +.el-switch__core:after { + content: ''; + position: absolute; + top: 1px; + left: 1px; + border-radius: 100%; + -webkit-transition: all 0.3s; + transition: all 0.3s; + width: 16px; + height: 16px; + background-color: #fff; +} + +.el-switch.is-checked .el-switch__core { + border-color: #00a5a7; + background-color: #00a5a7; +} + +.el-switch.is-checked .el-switch__core::after { + left: 100%; + margin-left: -17px; +} + +.el-switch.is-disabled { + opacity: 0.6; +} + +.el-switch--wide .el-switch__label.el-switch__label--left span { + left: 10px; +} + +.el-switch--wide .el-switch__label.el-switch__label--right span { + right: 10px; +} + +.el-switch .label-fade-enter, +.el-switch .label-fade-leave-active { + opacity: 0; +} + +.el-select-dropdown { + position: absolute; + z-index: 1001; + border: 1px solid #e4e7ed; + border-radius: 4px; + background-color: #fff; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + -webkit-box-sizing: border-box; + box-sizing: border-box; + margin: 5px 0; +} + +.el-select-dropdown.is-multiple .el-select-dropdown__item.selected { + color: #00a5a7; + background-color: #fff; +} + +.el-select-dropdown.is-multiple .el-select-dropdown__item.selected.hover { + background-color: #f5f7fa; +} + +.el-select-dropdown.is-multiple .el-select-dropdown__item.selected::after { + position: absolute; + right: 20px; + font-family: element-icons; + content: '\e6da'; + font-size: 12px; + font-weight: 700; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.el-select-dropdown .el-scrollbar.is-empty .el-select-dropdown__list { + padding: 0; +} + +.el-select-dropdown__empty { + padding: 10px 0; + margin: 0; + text-align: center; + color: #999; + font-size: 14px; +} + +.el-select-dropdown__wrap { + max-height: 274px; +} + +.el-select-dropdown__list { + list-style: none; + padding: 6px 0; + margin: 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-select-dropdown__item { + font-size: 14px; + padding: 0 20px; + position: relative; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + color: #606266; + height: 34px; + line-height: 34px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + cursor: pointer; +} + +.el-select .el-tag, +.el-table { + -webkit-box-sizing: border-box; +} + +.el-select-dropdown__item.is-disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-select-dropdown__item.is-disabled:hover { + background-color: #fff; +} + +.el-select-dropdown__item.hover, +.el-select-dropdown__item:hover { + background-color: #f5f7fa; +} + +.el-select-dropdown__item.selected { + color: #00a5a7; + font-weight: 700; +} + +.el-select-group { + margin: 0; + padding: 0; +} + +.el-select-group__wrap { + position: relative; + list-style: none; + margin: 0; + padding: 0; +} + +.el-select-group__wrap:not(:last-of-type) { + padding-bottom: 24px; +} + +.el-select-group__wrap:not(:last-of-type)::after { + content: ''; + position: absolute; + display: block; + left: 20px; + right: 20px; + bottom: 12px; + height: 1px; + background: #e4e7ed; +} + +.el-select-group__title { + padding-left: 20px; + font-size: 12px; + color: #909399; + line-height: 30px; +} + +.el-select-group .el-select-dropdown__item { + padding-left: 20px; +} + +.el-select { + display: inline-block; + position: relative; +} + +.el-select .el-select__tags > span { + display: contents; +} + +.el-select:hover .el-input__inner { + border-color: #c0c4cc; +} + +.el-select .el-input__inner { + cursor: pointer; + padding-right: 35px; +} + +.el-select .el-input__inner:focus { + border-color: #00a5a7; +} + +.el-select .el-input .el-select__caret { + color: #c0c4cc; + font-size: 14px; + -webkit-transition: -webkit-transform 0.3s; + transition: -webkit-transform 0.3s; + transition: transform 0.3s; + transition: transform 0.3s, -webkit-transform 0.3s; + -webkit-transform: rotateZ(180deg); + transform: rotateZ(180deg); + cursor: pointer; +} + +.el-select .el-input .el-select__caret.is-reverse { + -webkit-transform: rotateZ(0); + transform: rotateZ(0); +} + +.el-select .el-input .el-select__caret.is-show-close { + font-size: 14px; + text-align: center; + -webkit-transform: rotateZ(180deg); + transform: rotateZ(180deg); + border-radius: 100%; + color: #c0c4cc; + -webkit-transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); +} + +.el-select .el-input .el-select__caret.is-show-close:hover { + color: #909399; +} + +.el-select .el-input.is-disabled .el-input__inner { + cursor: not-allowed; +} + +.el-select .el-input.is-disabled .el-input__inner:hover { + border-color: #e4e7ed; +} + +.el-select .el-input.is-focus .el-input__inner { + border-color: #00a5a7; +} + +.el-select > .el-input { + display: block; +} + +.el-select__input { + border: none; + outline: 0; + padding: 0; + margin-left: 15px; + color: #666; + font-size: 14px; + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + height: 28px; + background-color: transparent; +} + +.el-select__input.is-mini { + height: 14px; +} + +.el-select__close { + cursor: pointer; + position: absolute; + top: 8px; + z-index: 1000; + right: 25px; + color: #c0c4cc; + line-height: 18px; + font-size: 14px; +} + +.el-select__close:hover { + color: #909399; +} + +.el-select__tags { + position: absolute; + line-height: normal; + white-space: normal; + z-index: 1; + top: 50%; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -ms-flex-wrap: wrap; + flex-wrap: wrap; +} + +.el-select .el-tag__close { + margin-top: -2px; +} + +.el-select .el-tag { + box-sizing: border-box; + border-color: transparent; + margin: 2px 0 2px 6px; + background-color: #f0f2f5; +} + +.el-select .el-tag__close.el-icon-close { + background-color: #c0c4cc; + right: -7px; + top: 0; + color: #fff; +} + +.el-select .el-tag__close.el-icon-close:hover { + background-color: #909399; +} + +.el-table, +.el-table__expanded-cell { + background-color: #fff; +} + +.el-select .el-tag__close.el-icon-close::before { + display: block; + -webkit-transform: translate(0, 0.5px); + transform: translate(0, 0.5px); +} + +.el-table { + position: relative; + overflow: hidden; + box-sizing: border-box; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + width: 100%; + max-width: 100%; + font-size: 14px; + color: #606266; +} + +.el-table--mini, +.el-table--small, +.el-table__expand-icon { + font-size: 12px; +} + +.el-table__empty-block { + min-height: 60px; + text-align: center; + width: 100%; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-table__empty-text { + line-height: 60px; + width: 50%; + color: #909399; +} + +.el-table__expand-column .cell { + padding: 0; + text-align: center; +} + +.el-table__expand-icon { + position: relative; + cursor: pointer; + color: #666; + -webkit-transition: -webkit-transform 0.2s ease-in-out; + transition: -webkit-transform 0.2s ease-in-out; + transition: transform 0.2s ease-in-out; + transition: transform 0.2s ease-in-out, -webkit-transform 0.2s ease-in-out; + height: 20px; +} + +.el-table__expand-icon--expanded { + -webkit-transform: rotate(90deg); + transform: rotate(90deg); +} + +.el-table__expand-icon > .el-icon { + position: absolute; + left: 50%; + top: 50%; + margin-left: -5px; + margin-top: -5px; +} + +.el-table__expanded-cell[class*='cell'] { + padding: 20px 50px; +} + +.el-table__expanded-cell:hover { + background-color: transparent !important; +} + +.el-table__placeholder { + display: inline-block; + width: 20px; +} + +.el-table__append-wrapper { + overflow: hidden; +} + +.el-table--fit { + border-right: 0; + border-bottom: 0; +} + +.el-table--fit td.gutter, +.el-table--fit th.gutter { + border-right-width: 1px; +} + +.el-table--scrollable-x .el-table__body-wrapper { + overflow-x: auto; +} + +.el-table--scrollable-y .el-table__body-wrapper { + overflow-y: auto; +} + +.el-table thead { + color: #909399; + font-weight: 500; +} + +.el-table thead.is-group th { + background: #f5f7fa; +} +.el-table tr { + background-color: #fff; +} + +.el-table td, +.el-table th { + padding: 1px 0; + min-width: 0; + height: 42px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + text-overflow: ellipsis; + vertical-align: middle; + position: relative; + text-align: left; +} + +.el-table td.is-center, +.el-table th.is-center { + text-align: center; +} + +.el-table td.is-right, +.el-table th.is-right { + text-align: right; +} + +.el-table td.gutter, +.el-table th.gutter { + width: 15px; + border-right-width: 0; + border-bottom-width: 0; + padding: 0; +} + +.el-table--medium td, +.el-table--medium th { + padding: 10px 0; +} + +.el-table--small td, +.el-table--small th { + padding: 8px 0; +} + +.el-table--mini td, +.el-table--mini th { + padding: 6px 0; +} + +.el-table .cell, +.el-table th div { + padding-right: 10px; + overflow: hidden; + text-overflow: ellipsis; +} + +.el-table .cell, +.el-table th div, +.el-table--border td:first-child .cell, +.el-table--border th:first-child .cell { + padding-left: 10px; +} + +.el-table tr input[type='checkbox'] { + margin: 0; +} + +.el-table td, +.el-table th.is-leaf { + border-bottom: 1px solid #ebeef5; +} + +.el-table th.is-leaf .cell { + white-space: nowrap; +} + +.el-table th.is-sortable { + cursor: pointer; +} + +.el-table th { + white-space: nowrap; + overflow: hidden; + -moz-user-select: none; + user-select: none; +} + +.el-table th div { + display: inline-block; + line-height: 40px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + white-space: nowrap; +} + +.el-table th > .cell { + position: relative; + word-wrap: normal; + text-overflow: ellipsis; + display: inline-block; + vertical-align: middle; + width: 100%; + -webkit-box-sizing: border-box; + box-sizing: border-box; + font-size: 14px; + color: #282b33; +} + +.el-table th > .cell.highlight { + color: #00a5a7; +} + +.el-table th.required > div::before { + display: inline-block; + content: ''; + width: 8px; + height: 8px; + border-radius: 50%; + background: #ff4d51; + margin-right: 5px; + vertical-align: middle; +} + +.el-table td div { + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-table td.gutter { + width: 0; +} + +.el-table .cell { + -webkit-box-sizing: border-box; + box-sizing: border-box; + white-space: normal; + word-break: break-all; + line-height: 23px; +} + +.el-table .cell.el-tooltip { + white-space: nowrap; + min-width: 50px; +} + +.el-table--border, +.el-table--group { + border: 1px solid #ebeef5; +} + +.el-table--border::after, +.el-table--group::after, +.el-table::before { + content: ''; + position: absolute; + background-color: #ebeef5; + z-index: 1; +} + +.el-table--border::after, +.el-table--group::after { + top: 0; + right: 0; + width: 1px; + height: 100%; +} + +.el-table::before { + left: 0; + bottom: 0; + width: 100%; + height: 1px; +} + +.el-table--border { + border-right: none; + border-bottom: none; +} + +.el-table--border.el-loading-parent--relative { + border-color: transparent; +} + +.el-table--border td, +.el-table--border th, +.el-table__body-wrapper .el-table--border.is-scrolling-left ~ .el-table__fixed { + border-right: 1px solid #ebeef5; +} + +.el-table--border th.gutter:last-of-type { + border-bottom: 1px solid #ebeef5; + border-bottom-width: 1px; +} + +.el-table--border th, +.el-table__fixed-right-patch { + border-bottom: 1px solid #ebeef5; +} + +.el-table__fixed, +.el-table__fixed-right { + position: absolute; + top: 0; + left: 0; + overflow-x: hidden; + overflow-y: hidden; + -webkit-box-shadow: 0 0 10px rgba(0, 0, 0, 0.12); + box-shadow: 0 0 10px rgba(0, 0, 0, 0.12); +} + +.el-table__fixed-right::before, +.el-table__fixed::before { + content: ''; + position: absolute; + left: 0; + bottom: 0; + width: 100%; + height: 1px; + background-color: #ebeef5; + z-index: 4; +} + +.el-table__fixed-right-patch { + position: absolute; + top: -1px; + right: 0; + background-color: #fff; +} + +.el-table__fixed-right { + top: 0; + left: auto; + right: 0; +} + +.el-table__fixed-right .el-table__fixed-body-wrapper, +.el-table__fixed-right .el-table__fixed-footer-wrapper, +.el-table__fixed-right .el-table__fixed-header-wrapper { + left: auto; + right: 0; +} + +.el-table__fixed-header-wrapper { + position: absolute; + left: 0; + top: 0; + z-index: 3; +} + +.el-table__fixed-footer-wrapper { + position: absolute; + left: 0; + bottom: 0; + z-index: 3; +} + +.el-table__fixed-footer-wrapper tbody td { + border-top: 1px solid #ebeef5; + background-color: #f5f7fa; + color: #606266; +} + +.el-table__fixed-body-wrapper { + position: absolute; + left: 0; + top: 37px; + overflow: hidden; + z-index: 3; +} + +.el-table__body-wrapper, +.el-table__footer-wrapper, +.el-table__header-wrapper { + width: 100%; +} + +.el-table__footer-wrapper { + margin-top: -1px; +} + +.el-table__footer-wrapper td { + border-top: 1px solid #ebeef5; +} + +.el-table__body, +.el-table__footer, +.el-table__header { + table-layout: fixed; + border-collapse: separate; +} + +.el-table__footer-wrapper, +.el-table__header-wrapper { + overflow: hidden; +} + +.el-table__footer-wrapper tbody td, +.el-table__header-wrapper tbody td { + background-color: #f5f7fa; + color: #606266; +} + +.el-table__body-wrapper { + overflow: hidden; + position: relative; +} + +.el-table__body-wrapper.is-scrolling-left ~ .el-table__fixed, +.el-table__body-wrapper.is-scrolling-none ~ .el-table__fixed, +.el-table__body-wrapper.is-scrolling-none ~ .el-table__fixed-right, +.el-table__body-wrapper.is-scrolling-right ~ .el-table__fixed-right { + -webkit-box-shadow: none; + box-shadow: none; +} + +.el-picker-panel, +.el-table-filter { + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-table__body-wrapper .el-table--border.is-scrolling-right ~ .el-table__fixed-right { + border-left: 1px solid #ebeef5; +} + +.el-table .caret-wrapper { + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-orient: vertical; + -webkit-box-direction: normal; + -ms-flex-direction: column; + flex-direction: column; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + height: 34px; + width: 24px; + vertical-align: middle; + cursor: pointer; + overflow: initial; + position: relative; +} + +.el-table .sort-caret { + width: 0; + height: 0; + border: 5px solid transparent; + position: absolute; + left: 7px; +} + +.el-table .sort-caret.ascending { + border-bottom-color: #c0c4cc; + top: 5px; +} + +.el-table .sort-caret.descending { + border-top-color: #c0c4cc; + bottom: 7px; +} + +.el-table .ascending .sort-caret.ascending { + border-bottom-color: #00a5a7; +} + +.el-table .descending .sort-caret.descending { + border-top-color: #00a5a7; +} + +.el-table .hidden-columns { + position: absolute; + z-index: -1; +} + +.el-table--striped .el-table__body tr.el-table__row--striped td { + background: #f7faff; +} + +.el-table--striped .el-table__body tr.el-table__row--striped.current-row td { + background-color: rgb(230, 246, 246); +} + +.el-table__body tr.hover-row.current-row > td, +.el-table__body tr.hover-row.el-table__row--striped.current-row > td, +.el-table__body tr.hover-row.el-table__row--striped > td, +.el-table__body tr.hover-row > td { + background-color: #f5f7fa; +} + +.el-table__body tr.current-row > td { + background-color: rgb(230, 246, 246); +} + +.el-table__column-resize-proxy { + position: absolute; + left: 200px; + top: 0; + bottom: 0; + width: 0; + border-left: 1px solid #ebeef5; + z-index: 10; +} + +.el-table__column-filter-trigger { + display: inline-block; + line-height: 34px; + cursor: pointer; +} + +.el-table__column-filter-trigger i { + color: #909399; + font-size: 12px; + -webkit-transform: scale(0.75); + transform: scale(0.75); +} + +.el-table--enable-row-transition .el-table__body td { + -webkit-transition: background-color 0.25s ease; + transition: background-color 0.25s ease; +} + +.el-table--enable-row-hover .el-table__body tr:hover > td { + background-color: #f5f7fa; +} + +.el-table--fluid-height .el-table__fixed, +.el-table--fluid-height .el-table__fixed-right { + bottom: 0; + overflow: hidden; +} + +.el-table [class*='el-table__row--level'] .el-table__expand-icon { + display: inline-block; + width: 20px; + line-height: 20px; + height: 20px; + text-align: center; + margin-right: 3px; +} + +.el-table-column--selection .cell { + padding-left: 14px; + padding-right: 14px; +} + +.el-table-filter { + border: 1px solid #ebeef5; + border-radius: 2px; + background-color: #fff; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + -webkit-box-sizing: border-box; + box-sizing: border-box; + margin: 2px 0; +} + +.el-date-table td, +.el-date-table td div { + height: 30px; + -webkit-box-sizing: border-box; +} + +.el-table-filter__list { + padding: 5px 0; + margin: 0; + list-style: none; + min-width: 100px; +} + +.el-table-filter__list-item { + line-height: 36px; + padding: 0 10px; + cursor: pointer; + font-size: 14px; +} + +.el-table-filter__list-item:hover { + background-color: rgb(230, 246, 246); + color: rgb(51, 183, 185); +} + +.el-table-filter__list-item.is-active { + background-color: #00a5a7; + color: #fff; +} + +.el-table-filter__content { + min-width: 100px; +} + +.el-table-filter__bottom { + border-top: 1px solid #ebeef5; + padding: 8px; +} + +.el-table-filter__bottom button { + background: 0 0; + border: none; + color: #606266; + cursor: pointer; + font-size: 13px; + padding: 0 3px; +} + +.el-date-table td.in-range div, +.el-date-table td.in-range div:hover, +.el-date-table.is-week-mode .el-date-table__row.current div, +.el-date-table.is-week-mode .el-date-table__row:hover div { + background-color: #f2f6fc; +} + +.el-table-filter__bottom button:hover { + color: #00a5a7; +} + +.el-table-filter__bottom button:focus { + outline: 0; +} + +.el-table-filter__bottom button.is-disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-table-filter__wrap { + max-height: 280px; +} + +.el-table-filter__checkbox-group { + padding: 10px; +} + +.el-table-filter__checkbox-group label.el-checkbox { + display: block; + margin-right: 5px; + margin-bottom: 8px; + margin-left: 5px; +} + +.el-table-filter__checkbox-group .el-checkbox:last-child { + margin-bottom: 0; +} + +.el-date-table { + font-size: 12px; + -moz-user-select: none; + user-select: none; +} + +.el-slider__button-wrapper, +.el-time-panel { + -webkit-user-select: none; + -ms-user-select: none; +} + +.el-date-table.is-week-mode .el-date-table__row:hover td.available:hover { + color: #606266; +} + +.el-date-table.is-week-mode .el-date-table__row:hover td:first-child div { + margin-left: 5px; + border-top-left-radius: 15px; + border-bottom-left-radius: 15px; +} + +.el-date-table.is-week-mode .el-date-table__row:hover td:last-child div { + margin-right: 5px; + border-top-right-radius: 15px; + border-bottom-right-radius: 15px; +} + +.el-date-table td { + width: 32px; + padding: 4px 0; + box-sizing: border-box; + text-align: center; + cursor: pointer; + position: relative; +} + +.el-date-table td div { + padding: 3px 0; + box-sizing: border-box; +} + +.el-date-table td span { + width: 24px; + height: 24px; + display: block; + margin: 0 auto; + line-height: 24px; + position: absolute; + left: 50%; + -webkit-transform: translateX(-50%); + transform: translateX(-50%); + border-radius: 50%; +} + +.el-date-table td.next-month, +.el-date-table td.prev-month { + color: #c0c4cc; +} + +.el-date-table td.today { + position: relative; +} + +.el-date-table td.today span { + color: #00a5a7; + font-weight: 700; +} + +.el-date-table td.today.end-date span, +.el-date-table td.today.start-date span { + color: #fff; +} + +.el-date-table td.available:hover { + color: #00a5a7; +} + +.el-date-table td.current:not(.disabled) span { + color: #fff; + background-color: #00a5a7; +} + +.el-date-table td.end-date div, +.el-date-table td.start-date div { + color: #fff; +} + +.el-date-table td.end-date span, +.el-date-table td.start-date span { + background-color: #00a5a7; +} + +.el-date-table td.start-date div { + margin-left: 5px; + border-top-left-radius: 15px; + border-bottom-left-radius: 15px; +} + +.el-date-table td.end-date div { + margin-right: 5px; + border-top-right-radius: 15px; + border-bottom-right-radius: 15px; +} + +.el-date-table td.disabled div { + background-color: #f5f7fa; + opacity: 1; + cursor: not-allowed; + color: #c0c4cc; +} + +.el-date-table td.selected div { + margin-left: 5px; + margin-right: 5px; + background-color: #f2f6fc; + border-radius: 15px; +} + +.el-date-table td.selected div:hover { + background-color: #f2f6fc; +} + +.el-date-table td.selected span { + background-color: #00a5a7; + color: #fff; + border-radius: 15px; +} + +.el-date-table td.week { + font-size: 80%; + color: #606266; +} + +.el-month-table, +.el-year-table { + font-size: 12px; + border-collapse: collapse; +} + +.el-date-table th { + padding: 5px; + color: #606266; + font-weight: 400; + border-bottom: solid 1px #ebeef5; +} + +.el-month-table { + margin: -1px; +} + +.el-month-table td { + text-align: center; + padding: 8px 0; + cursor: pointer; +} + +.el-month-table td div { + height: 48px; + padding: 6px 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-month-table td.today .cell { + color: #00a5a7; + font-weight: 700; +} + +.el-month-table td.today.end-date .cell, +.el-month-table td.today.start-date .cell { + color: #fff; +} + +.el-month-table td.disabled .cell { + background-color: #f5f7fa; + cursor: not-allowed; + color: #c0c4cc; +} + +.el-month-table td.disabled .cell:hover { + color: #c0c4cc; +} + +.el-month-table td .cell { + width: 60px; + height: 36px; + display: block; + line-height: 36px; + color: #606266; + margin: 0 auto; + border-radius: 18px; +} + +.el-month-table td .cell:hover { + color: #00a5a7; +} + +.el-month-table td.in-range div, +.el-month-table td.in-range div:hover { + background-color: #f2f6fc; +} + +.el-month-table td.end-date div, +.el-month-table td.start-date div { + color: #fff; +} + +.el-month-table td.end-date .cell, +.el-month-table td.start-date .cell { + color: #fff; + background-color: #00a5a7; +} + +.el-month-table td.start-date div { + border-top-left-radius: 24px; + border-bottom-left-radius: 24px; +} + +.el-month-table td.end-date div { + border-top-right-radius: 24px; + border-bottom-right-radius: 24px; +} + +.el-month-table td.current:not(.disabled) .cell { + color: #00a5a7; +} + +.el-year-table { + margin: -1px; +} + +.el-year-table .el-icon { + color: #303133; +} + +.el-year-table td { + text-align: center; + padding: 20px 3px; + cursor: pointer; +} + +.el-year-table td.today .cell { + color: #00a5a7; + font-weight: 700; +} + +.el-year-table td.disabled .cell { + background-color: #f5f7fa; + cursor: not-allowed; + color: #c0c4cc; +} + +.el-year-table td.disabled .cell:hover { + color: #c0c4cc; +} + +.el-year-table td .cell { + width: 48px; + height: 32px; + display: block; + line-height: 32px; + color: #606266; + margin: 0 auto; +} + +.el-year-table td .cell:hover, +.el-year-table td.current:not(.disabled) .cell { + color: #00a5a7; +} + +.el-date-range-picker { + width: 646px; +} + +.el-date-range-picker.has-sidebar { + width: 756px; +} + +.el-date-range-picker table { + table-layout: fixed; + width: 100%; +} + +.el-date-range-picker .el-picker-panel__body { + min-width: 513px; +} + +.el-date-range-picker .el-picker-panel__content { + margin: 0; +} + +.el-date-range-picker__header { + position: relative; + text-align: center; + height: 28px; +} + +.el-date-range-picker__header [class*='arrow-left'] { + float: left; +} + +.el-date-range-picker__header [class*='arrow-right'] { + float: right; +} + +.el-date-range-picker__header div { + font-size: 16px; + font-weight: 500; + margin-right: 50px; +} + +.el-date-range-picker__content { + float: left; + width: 50%; + -webkit-box-sizing: border-box; + box-sizing: border-box; + margin: 0; + padding: 16px; +} + +.el-date-range-picker__content.is-left { + border-right: 1px solid #e4e4e4; +} + +.el-date-range-picker__content .el-date-range-picker__header div { + margin-left: 50px; + margin-right: 50px; +} + +.el-date-range-picker__editors-wrap { + -webkit-box-sizing: border-box; + box-sizing: border-box; + display: table-cell; +} + +.el-date-range-picker__editors-wrap.is-right { + text-align: right; +} + +.el-date-range-picker__time-header { + position: relative; + border-bottom: 1px solid #e4e4e4; + font-size: 12px; + padding: 8px 5px 5px; + display: table; + width: 100%; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-date-range-picker__time-header > .el-icon-arrow-right { + font-size: 20px; + vertical-align: middle; + display: table-cell; + color: #303133; +} + +.el-date-range-picker__time-picker-wrap { + position: relative; + display: table-cell; + padding: 0 5px; +} + +.el-date-range-picker__time-picker-wrap .el-picker-panel { + position: absolute; + top: 13px; + right: 0; + z-index: 1; + background: #fff; +} + +.el-date-picker { + width: 322px; +} + +.el-date-picker.has-sidebar.has-time { + width: 434px; +} + +.el-date-picker.has-sidebar { + width: 438px; +} + +.el-date-picker.has-time .el-picker-panel__body-wrapper { + position: relative; +} + +.el-date-picker .el-picker-panel__content { + width: 292px; +} + +.el-date-picker table { + table-layout: fixed; + width: 100%; +} + +.el-date-picker__editor-wrap { + position: relative; + display: table-cell; + padding: 0 5px; +} + +.el-date-picker__time-header { + position: relative; + border-bottom: 1px solid #e4e4e4; + font-size: 12px; + padding: 8px 5px 5px; + display: table; + width: 100%; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-date-picker__header { + margin: 12px; + text-align: center; +} + +.el-date-picker__header--bordered { + margin-bottom: 0; + padding-bottom: 12px; + border-bottom: solid 1px #ebeef5; +} + +.el-date-picker__header--bordered + .el-picker-panel__content { + margin-top: 0; +} + +.el-date-picker__header-label { + font-size: 16px; + font-weight: 500; + padding: 0 5px; + line-height: 22px; + text-align: center; + cursor: pointer; + color: #606266; +} + +.el-date-picker__header-label.active, +.el-date-picker__header-label:hover { + color: #00a5a7; +} + +.el-date-picker__prev-btn { + float: left; +} + +.el-date-picker__next-btn { + float: right; +} + +.el-date-picker__time-wrap { + padding: 10px; + text-align: center; +} + +.el-date-picker__time-label { + float: left; + cursor: pointer; + line-height: 30px; + margin-left: 10px; +} + +.time-select { + margin: 5px 0; + min-width: 0; +} + +.time-select .el-picker-panel__content { + max-height: 200px; + margin: 0; +} + +.time-select-item { + padding: 8px 10px; + font-size: 14px; + line-height: 20px; +} + +.time-select-item.selected:not(.disabled) { + color: #00a5a7; + font-weight: 700; +} + +.time-select-item.disabled { + color: #e4e7ed; + cursor: not-allowed; +} + +.time-select-item:hover { + background-color: #f5f7fa; + font-weight: 700; + cursor: pointer; +} + +.el-date-editor { + position: relative; + display: inline-block; + text-align: left; +} + +.el-date-editor.el-input, +.el-date-editor.el-input__inner { + width: 220px; +} + +.el-date-editor--monthrange.el-input, +.el-date-editor--monthrange.el-input__inner { + width: 300px; +} + +.el-date-editor--daterange.el-input, +.el-date-editor--daterange.el-input__inner, +.el-date-editor--timerange.el-input, +.el-date-editor--timerange.el-input__inner { + width: 350px; +} + +.el-date-editor--datetimerange.el-input, +.el-date-editor--datetimerange.el-input__inner { + width: 400px; +} + +.el-date-editor--dates .el-input__inner { + text-overflow: ellipsis; + white-space: nowrap; +} + +.el-date-editor .el-icon-circle-close { + cursor: pointer; +} + +.el-date-editor .el-range__icon { + font-size: 14px; + margin-left: -5px; + color: #c0c4cc; + float: left; + line-height: 32px; +} + +.el-date-editor .el-range-input, +.el-date-editor .el-range-separator { + height: 100%; + margin: 0; + text-align: center; + display: inline-block; + font-size: 14px; +} + +.el-date-editor .el-range-input { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + border: none; + outline: 0; + padding: 0; + width: 39%; + color: #606266; +} + +.el-date-editor .el-range-input::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-date-editor .el-range-input:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-date-editor .el-range-input::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-date-editor .el-range-input::placeholder { + color: #c0c4cc; +} + +.el-date-editor .el-range-separator { + padding: 0 5px; + line-height: 32px; + width: 5%; + color: #303133; +} + +.el-date-editor .el-range__close-icon { + font-size: 14px; + color: #c0c4cc; + width: 25px; + display: inline-block; + float: right; + line-height: 32px; +} + +.el-range-editor.el-input__inner { + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + padding: 3px 10px; +} + +.el-range-editor .el-range-input { + line-height: 1; +} + +.el-range-editor.is-active, +.el-range-editor.is-active:hover { + border-color: #00a5a7; +} + +.el-range-editor--medium.el-input__inner { + height: 36px; +} + +.el-range-editor--medium .el-range-separator { + line-height: 28px; + font-size: 14px; +} + +.el-range-editor--medium .el-range-input { + font-size: 14px; +} + +.el-range-editor--medium .el-range__close-icon, +.el-range-editor--medium .el-range__icon { + line-height: 28px; +} + +.el-range-editor--small.el-input__inner { + height: 32px; +} + +.el-range-editor--small .el-range-separator { + line-height: 24px; + font-size: 13px; +} + +.el-range-editor--small .el-range-input { + font-size: 13px; +} + +.el-range-editor--small .el-range__close-icon, +.el-range-editor--small .el-range__icon { + line-height: 24px; +} + +.el-range-editor--mini.el-input__inner { + height: 28px; +} + +.el-range-editor--mini .el-range-separator { + line-height: 20px; + font-size: 12px; +} + +.el-range-editor--mini .el-range-input { + font-size: 12px; +} + +.el-range-editor--mini .el-range__close-icon, +.el-range-editor--mini .el-range__icon { + line-height: 20px; +} + +.el-range-editor.is-disabled { + background-color: #f5f7fa; + border-color: #e4e7ed; + color: #c0c4cc; + cursor: not-allowed; +} + +.el-range-editor.is-disabled:focus, +.el-range-editor.is-disabled:hover { + border-color: #e4e7ed; +} + +.el-range-editor.is-disabled input { + background-color: #f5f7fa; + color: #c0c4cc; + cursor: not-allowed; +} + +.el-range-editor.is-disabled input::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-range-editor.is-disabled input:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-range-editor.is-disabled input::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-range-editor.is-disabled input::placeholder { + color: #c0c4cc; +} + +.el-range-editor.is-disabled .el-range-separator { + color: #c0c4cc; +} + +.el-picker-panel { + color: #606266; + border: 1px solid #e4e7ed; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + background: #fff; + border-radius: 4px; + line-height: 30px; + margin: 5px 0; +} + +.el-popover, +.el-time-panel { + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-picker-panel__body-wrapper::after, +.el-picker-panel__body::after { + content: ''; + display: table; + clear: both; +} + +.el-picker-panel__content { + position: relative; + margin: 15px; +} + +.el-picker-panel__footer { + border-top: 1px solid #e4e4e4; + padding: 4px; + text-align: right; + background-color: #fff; + position: relative; + font-size: 0; +} + +.el-picker-panel__shortcut { + display: block; + width: 100%; + border: 0; + background-color: transparent; + line-height: 28px; + font-size: 14px; + color: #606266; + padding-left: 12px; + text-align: left; + outline: 0; + cursor: pointer; +} + +.el-picker-panel__shortcut:hover { + color: #00a5a7; +} + +.el-picker-panel__shortcut.active { + background-color: #e6f1fe; + color: #00a5a7; +} + +.el-picker-panel__btn { + border: 1px solid #dcdcdc; + color: #333; + line-height: 24px; + border-radius: 2px; + padding: 0 20px; + cursor: pointer; + background-color: transparent; + outline: 0; + font-size: 12px; +} + +.el-picker-panel__btn[disabled] { + color: #ccc; + cursor: not-allowed; +} + +.el-picker-panel__icon-btn { + font-size: 12px; + color: #303133; + border: 0; + background: 0 0; + cursor: pointer; + outline: 0; + margin-top: 8px; +} + +.el-picker-panel__icon-btn:hover { + color: #00a5a7; +} + +.el-picker-panel__icon-btn.is-disabled { + color: #bbb; +} + +.el-picker-panel__icon-btn.is-disabled:hover { + cursor: not-allowed; +} + +.el-picker-panel__link-btn { + vertical-align: middle; +} + +.el-picker-panel [slot='sidebar'], +.el-picker-panel__sidebar { + position: absolute; + top: 0; + bottom: 0; + width: 110px; + border-right: 1px solid #e4e4e4; + -webkit-box-sizing: border-box; + box-sizing: border-box; + padding-top: 6px; + background-color: #fff; + overflow: auto; +} + +.el-picker-panel [slot='sidebar'] + .el-picker-panel__body, +.el-picker-panel__sidebar + .el-picker-panel__body { + margin-left: 110px; +} + +.el-time-spinner.has-seconds .el-time-spinner__wrapper { + width: 33.3%; +} + +.el-time-spinner__wrapper { + max-height: 190px; + overflow: auto; + display: inline-block; + width: 50%; + vertical-align: top; + position: relative; +} + +.el-time-spinner__wrapper .el-scrollbar__wrap:not(.el-scrollbar__wrap--hidden-default) { + padding-bottom: 15px; +} + +.el-time-spinner__input.el-input .el-input__inner, +.el-time-spinner__list { + padding: 0; + text-align: center; +} + +.el-time-spinner__wrapper.is-arrow { + -webkit-box-sizing: border-box; + box-sizing: border-box; + text-align: center; + overflow: hidden; +} + +.el-time-spinner__wrapper.is-arrow .el-time-spinner__list { + -webkit-transform: translateY(-32px); + transform: translateY(-32px); +} + +.el-time-spinner__wrapper.is-arrow .el-time-spinner__item:hover:not(.disabled):not(.active) { + background: #fff; + cursor: default; +} + +.el-time-spinner__arrow { + font-size: 12px; + color: #909399; + position: absolute; + left: 0; + width: 100%; + z-index: 1; + text-align: center; + height: 30px; + line-height: 30px; + cursor: pointer; +} + +.el-time-spinner__arrow:hover { + color: #00a5a7; +} + +.el-time-spinner__arrow.el-icon-arrow-up { + top: 10px; +} + +.el-time-spinner__arrow.el-icon-arrow-down { + bottom: 10px; +} + +.el-time-spinner__input.el-input { + width: 70%; +} + +.el-time-spinner__list { + margin: 0; + list-style: none; +} + +.el-time-spinner__list::after, +.el-time-spinner__list::before { + content: ''; + display: block; + width: 100%; + height: 80px; +} + +.el-time-spinner__item { + height: 32px; + line-height: 32px; + font-size: 12px; + color: #606266; +} + +.el-time-spinner__item:hover:not(.disabled):not(.active) { + background: #f5f7fa; + cursor: pointer; +} + +.el-time-spinner__item.active:not(.disabled) { + color: #303133; + font-weight: 700; +} + +.el-time-spinner__item.disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-time-panel { + margin: 5px 0; + border: 1px solid #e4e7ed; + background-color: #fff; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + border-radius: 2px; + position: absolute; + width: 180px; + left: 0; + z-index: 1000; + -moz-user-select: none; + user-select: none; + -webkit-box-sizing: content-box; + box-sizing: content-box; +} + +.el-time-panel__content { + font-size: 0; + position: relative; + overflow: hidden; +} + +.el-time-panel__content::after, +.el-time-panel__content::before { + content: ''; + top: 50%; + position: absolute; + margin-top: -15px; + height: 32px; + z-index: -1; + left: 0; + right: 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; + padding-top: 6px; + text-align: left; + border-top: 1px solid #e4e7ed; + border-bottom: 1px solid #e4e7ed; +} + +.el-time-panel__content::after { + left: 50%; + margin-left: 12%; + margin-right: 12%; +} + +.el-time-panel__content::before { + padding-left: 50%; + margin-right: 12%; + margin-left: 12%; +} + +.el-time-panel__content.has-seconds::after { + left: calc(100% / 3 * 2); +} + +.el-time-panel__content.has-seconds::before { + padding-left: calc(100% / 3); +} + +.el-time-panel__footer { + border-top: 1px solid #e4e4e4; + padding: 4px; + height: 36px; + line-height: 25px; + text-align: right; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-time-panel__btn { + border: none; + line-height: 28px; + padding: 0 5px; + margin: 0 5px; + cursor: pointer; + background-color: transparent; + outline: 0; + font-size: 12px; + color: #303133; +} + +.el-time-panel__btn.confirm { + font-weight: 800; + color: #00a5a7; +} + +.el-time-range-picker { + width: 354px; + overflow: visible; +} + +.el-time-range-picker__content { + position: relative; + text-align: center; + padding: 10px; +} + +.el-time-range-picker__cell { + -webkit-box-sizing: border-box; + box-sizing: border-box; + margin: 0; + padding: 4px 7px 7px; + width: 50%; + display: inline-block; +} + +.el-time-range-picker__header { + margin-bottom: 5px; + text-align: center; + font-size: 14px; +} + +.el-time-range-picker__body { + border-radius: 2px; + border: 1px solid #e4e7ed; +} + +.el-popover { + position: absolute; + background: #fff; + min-width: 150px; + border: 1px solid #ebeef5; + padding: 12px; + z-index: 2000; + color: #606266; + line-height: 1.4; + text-align: justify; + font-size: 14px; + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + word-break: break-all; +} + +.el-popover--plain { + padding: 18px 20px; +} + +.el-popover__title { + color: #303133; + font-size: 16px; + line-height: 1; + margin-bottom: 12px; +} + +.v-modal-enter { + -webkit-animation: v-modal-in 0.2s ease; + animation: v-modal-in 0.2s ease; +} + +.v-modal-leave { + -webkit-animation: v-modal-out 0.2s ease forwards; + animation: v-modal-out 0.2s ease forwards; +} + +@keyframes v-modal-in { + 0% { + opacity: 0; + } +} + +@keyframes v-modal-out { + 100% { + opacity: 0; + } +} + +.v-modal { + position: fixed; + left: 0; + top: 0; + width: 100%; + height: 100%; + opacity: 0.5; + background: #000; +} + +.el-popup-parent--hidden { + overflow: hidden; +} + +.el-message-box { + display: inline-block; + width: 420px; + padding-bottom: 10px; + vertical-align: middle; + background-color: #fff; + border-radius: 4px; + border: 1px solid #ebeef5; + font-size: 18px; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + text-align: left; + overflow: hidden; + -webkit-backface-visibility: hidden; + backface-visibility: hidden; +} + +.el-message-box__wrapper { + position: fixed; + top: 0; + bottom: 0; + left: 0; + right: 0; + text-align: center; +} + +.el-message-box__wrapper::after { + content: ''; + display: inline-block; + height: 100%; + width: 0; + vertical-align: middle; +} + +.el-message-box__header { + position: relative; + padding: 15px 15px 10px; +} + +.el-message-box__title { + padding-left: 0; + margin-bottom: 0; + font-size: 18px; + line-height: 1; + color: #303133; +} + +.el-message-box__headerbtn { + position: absolute; + top: 15px; + right: 15px; + padding: 0; + border: none; + outline: 0; + background: 0 0; + font-size: 16px; + cursor: pointer; +} + +.el-form-item.is-error .el-input__inner, +.el-form-item.is-error .el-input__inner:focus, +.el-form-item.is-error .el-textarea__inner, +.el-form-item.is-error .el-textarea__inner:focus, +.el-message-box__input input.invalid, +.el-message-box__input input.invalid:focus { + border-color: #f56c6c; +} + +.el-message-box__headerbtn .el-message-box__close { + color: #909399; +} + +.el-message-box__headerbtn:focus .el-message-box__close, +.el-message-box__headerbtn:hover .el-message-box__close { + color: #00a5a7; +} + +.el-message-box__content { + position: relative; + padding: 10px 15px; + color: #606266; + font-size: 14px; +} + +.el-message-box__input { + padding-top: 15px; +} + +.el-message-box__status { + position: absolute; + top: 50%; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + font-size: 24px !important; +} + +.el-message-box__status::before { + padding-left: 1px; +} + +.el-message-box__status + .el-message-box__message { + padding-left: 36px; + padding-right: 12px; +} + +.el-message-box__status.el-icon-success { + color: #67c23a; +} + +.el-message-box__status.el-icon-info { + color: #909399; +} + +.el-message-box__status.el-icon-warning { + color: #e6a23c; +} + +.el-message-box__status.el-icon-error { + color: #f56c6c; +} + +.el-message-box__message { + margin: 0; +} + +.el-message-box__message p { + margin: 0; + line-height: 24px; +} + +.el-message-box__errormsg { + color: #f56c6c; + font-size: 12px; + min-height: 18px; + margin-top: 2px; +} + +.el-message-box__btns { + padding: 5px 15px 0; + text-align: right; +} + +.el-message-box__btns button:nth-child(2) { + margin-left: 10px; +} + +.el-message-box__btns-reverse { + -webkit-box-orient: horizontal; + -webkit-box-direction: reverse; + -ms-flex-direction: row-reverse; + flex-direction: row-reverse; +} + +.el-container, +.el-container.is-vertical, +.el-drawer, +.el-link, +.el-steps--vertical { + -webkit-box-direction: normal; +} + +.el-message-box--center { + padding-bottom: 30px; +} + +.el-message-box--center .el-message-box__header { + padding-top: 30px; +} + +.el-message-box--center .el-message-box__title { + position: relative; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; +} + +.el-message-box--center .el-message-box__status { + position: relative; + top: auto; + padding-right: 5px; + text-align: center; + -webkit-transform: translateY(-1px); + transform: translateY(-1px); +} + +.el-message-box--center .el-message-box__message { + margin-left: 0; +} + +.el-message-box--center .el-message-box__btns, +.el-message-box--center .el-message-box__content { + text-align: center; +} + +.el-message-box--center .el-message-box__content { + padding-left: 27px; + padding-right: 27px; +} + +.msgbox-fade-enter-active { + -webkit-animation: msgbox-fade-in 0.3s; + animation: msgbox-fade-in 0.3s; +} + +.msgbox-fade-leave-active { + -webkit-animation: msgbox-fade-out 0.3s; + animation: msgbox-fade-out 0.3s; +} + +@-webkit-keyframes msgbox-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@keyframes msgbox-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@-webkit-keyframes msgbox-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +@keyframes msgbox-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +.el-breadcrumb { + font-size: 14px; + line-height: 1; +} + +.el-breadcrumb::after, +.el-breadcrumb::before { + display: table; + content: ''; +} + +.el-breadcrumb::after { + clear: both; +} + +.el-breadcrumb__separator { + margin: 0 9px; + font-weight: 700; + color: #c0c4cc; +} + +.el-breadcrumb__separator[class*='icon'] { + margin: 0 6px; + font-weight: 400; +} + +.el-breadcrumb__item { + float: left; +} + +.el-breadcrumb__inner { + color: #606266; +} + +.el-breadcrumb__inner a, +.el-breadcrumb__inner.is-link { + font-weight: 700; + text-decoration: none; + -webkit-transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + color: #303133; +} + +.el-breadcrumb__inner a:hover, +.el-breadcrumb__inner.is-link:hover { + color: #00a5a7; + cursor: pointer; +} + +.el-breadcrumb__item:last-child .el-breadcrumb__inner, +.el-breadcrumb__item:last-child .el-breadcrumb__inner a, +.el-breadcrumb__item:last-child .el-breadcrumb__inner a:hover { + font-weight: 400; + color: #606266; + cursor: text; +} + +.el-breadcrumb__item:last-child .el-breadcrumb__separator { + display: none; +} + +.el-form--label-left .el-form-item__label { + text-align: left; +} + +.el-form--label-top .el-form-item__label { + float: none; + display: inline-block; + text-align: left; + padding: 0 0 10px; +} + +.el-form--inline .el-form-item { + display: inline-block; + margin-right: 10px; + vertical-align: top; +} + +.el-form--inline .el-form-item__label { + float: none; + display: inline-block; +} + +.el-form--inline .el-form-item__content { + display: inline-block; + vertical-align: top; +} + +.el-form--inline.el-form--label-top .el-form-item__content { + display: block; +} + +.el-form-item { + margin-bottom: 22px; +} + +.el-form-item::after, +.el-form-item::before { + display: table; + content: ''; +} + +.el-form-item::after { + clear: both; +} + +.el-form-item .el-form-item { + margin-bottom: 0; +} + +.el-form-item--mini.el-form-item, +.el-form-item--small.el-form-item { + margin-bottom: 18px; +} + +.el-form-item .el-input__validateIcon { + display: none; +} + +.el-form-item--medium .el-form-item__content, +.el-form-item--medium .el-form-item__label { + line-height: 36px; +} + +.el-form-item--small .el-form-item__content, +.el-form-item--small .el-form-item__label { + line-height: 32px; +} + +.el-form-item--small .el-form-item__error { + padding-top: 2px; +} + +.el-form-item--mini .el-form-item__content, +.el-form-item--mini .el-form-item__label { + line-height: 28px; +} + +.el-form-item--mini .el-form-item__error { + padding-top: 1px; +} + +.el-form-item__label-wrap { + float: left; +} + +.el-form-item__label-wrap .el-form-item__label { + display: inline-block; + float: none; +} + +.el-form-item__label { + text-align: right; + vertical-align: middle; + float: left; + font-size: 14px; + color: #606266; + line-height: 40px; + padding: 0 12px 0 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-form-item__content { + line-height: 40px; + position: relative; + font-size: 14px; +} + +.el-form-item__content::after, +.el-form-item__content::before { + display: table; + content: ''; +} + +.el-form-item__content::after { + clear: both; +} + +.el-form-item__content .el-input-group { + vertical-align: top; +} + +.el-form-item__error { + color: #f56c6c; + font-size: 12px; + line-height: 1; + padding-top: 4px; + position: absolute; + top: 100%; + left: 0; + text-align: left; +} + +.el-form-item__error--inline { + position: relative; + top: auto; + left: auto; + display: inline-block; + margin-left: 10px; +} + +.el-form-item.is-required:not(.is-no-asterisk) + .el-form-item__label-wrap + > .el-form-item__label:before, +.el-form-item.is-required:not(.is-no-asterisk) > .el-form-item__label:before { + content: '*'; + color: #f56c6c; + margin-right: 4px; +} + +.el-form-item.is-error .el-input-group__append .el-input__inner, +.el-form-item.is-error .el-input-group__prepend .el-input__inner { + border-color: transparent; +} + +.el-form-item.is-error .el-input__validateIcon { + color: #f56c6c; +} + +.el-form-item--feedback .el-input__validateIcon { + display: inline-block; +} + +.el-tabs__header { + padding: 0; + position: relative; + margin: 0 0 15px; +} + +.el-tabs__active-bar { + position: absolute; + bottom: 0; + left: 0; + height: 2px; + background-color: #00a5a7; + z-index: 1; + -webkit-transition: -webkit-transform 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: -webkit-transform 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: transform 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: transform 0.3s cubic-bezier(0.645, 0.045, 0.355, 1), + -webkit-transform 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + list-style: none; +} + +.el-tabs__new-tab { + float: right; + border: 1px solid #d3dce6; + height: 18px; + width: 18px; + line-height: 18px; + margin: 12px 0 9px 10px; + border-radius: 3px; + text-align: center; + font-size: 12px; + color: #d3dce6; + cursor: pointer; + -webkit-transition: all 0.15s; + transition: all 0.15s; +} + +.el-collapse-item__arrow, +.el-tabs__nav { + -webkit-transition: -webkit-transform 0.3s; +} + +.el-tabs__new-tab .el-icon-plus { + -webkit-transform: scale(0.8, 0.8); + transform: scale(0.8, 0.8); +} + +.el-tabs__new-tab:hover { + color: #00a5a7; +} + +.el-tabs__nav-wrap { + overflow: hidden; + margin-bottom: -1px; + position: relative; +} + +.el-tabs__nav-wrap::after { + content: ''; + position: absolute; + left: 0; + bottom: 0; + width: 100%; + height: 2px; + background-color: #e4e7ed; + z-index: 1; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__nav-wrap::after, +.el-tabs--card > .el-tabs__header .el-tabs__nav-wrap::after { + content: none; +} + +.el-tabs__nav-wrap.is-scrollable { + padding: 0 20px; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-tabs__nav-scroll { + overflow: hidden; +} + +.el-tabs__nav-next, +.el-tabs__nav-prev { + position: absolute; + cursor: pointer; + line-height: 44px; + font-size: 12px; + color: #909399; +} + +.el-tabs__nav-next { + right: 0; +} + +.el-tabs__nav-prev { + left: 0; +} + +.el-tabs__nav { + white-space: nowrap; + position: relative; + transition: -webkit-transform 0.3s; + transition: transform 0.3s; + transition: transform 0.3s, -webkit-transform 0.3s; + float: left; + z-index: 2; +} + +.el-tabs__nav.is-stretch { + min-width: 100%; + display: -webkit-box; + display: -ms-flexbox; + display: flex; +} + +.el-tabs__nav.is-stretch > * { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + text-align: center; +} + +.el-tabs__item { + padding: 0 20px; + height: 40px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + line-height: 40px; + display: inline-block; + list-style: none; + font-size: 14px; + font-weight: 500; + color: #303133; + position: relative; +} + +.el-tabs__item:focus, +.el-tabs__item:focus:active { + outline: 0; +} + +.el-tabs__item:focus.is-active.is-focus:not(:active) { + -webkit-box-shadow: 0 0 2px 2px #00a5a7 inset; + box-shadow: 0 0 2px 2px #00a5a7 inset; + border-radius: 3px; +} + +.el-tabs__item .el-icon-close { + border-radius: 50%; + text-align: center; + -webkit-transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + margin-left: 5px; +} + +.el-tabs__item .el-icon-close:before { + -webkit-transform: scale(0.9); + transform: scale(0.9); + display: inline-block; +} + +.el-tabs__item .el-icon-close:hover { + background-color: #c0c4cc; + color: #fff; +} + +.el-tabs__item.is-active { + color: #00a5a7; +} + +.el-tabs__item:hover { + color: #00a5a7; + cursor: pointer; +} + +.el-tabs__item.is-disabled { + color: #c0c4cc; + cursor: default; +} + +.el-tabs__content { + overflow: hidden; + position: relative; +} + +.el-tabs--card > .el-tabs__header { + border-bottom: 1px solid #e4e7ed; +} + +.el-tabs--card > .el-tabs__header .el-tabs__nav { + border: 1px solid #e4e7ed; + border-bottom: none; + border-radius: 4px 4px 0 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-tabs--card > .el-tabs__header .el-tabs__active-bar { + display: none; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item .el-icon-close { + position: relative; + font-size: 12px; + width: 0; + height: 14px; + vertical-align: middle; + line-height: 15px; + overflow: hidden; + top: -1px; + right: -2px; + -webkit-transform-origin: 100% 50%; + transform-origin: 100% 50%; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item.is-active.is-closable .el-icon-close, +.el-tabs--card > .el-tabs__header .el-tabs__item.is-closable:hover .el-icon-close { + width: 14px; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item { + border-bottom: 1px solid transparent; + border-left: 1px solid #e4e7ed; + -webkit-transition: color 0.3s cubic-bezier(0.645, 0.045, 0.355, 1), + padding 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: color 0.3s cubic-bezier(0.645, 0.045, 0.355, 1), + padding 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); +} + +.el-tabs--card > .el-tabs__header .el-tabs__item:first-child { + border-left: none; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item.is-closable:hover { + padding-left: 13px; + padding-right: 13px; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item.is-active { + border-bottom-color: #fff; +} + +.el-tabs--card > .el-tabs__header .el-tabs__item.is-active.is-closable { + padding-left: 20px; + padding-right: 20px; +} + +.el-tabs--border-card { + background: #fff; + border: 1px solid #dcdfe6; + -webkit-box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.12), 0 0 6px 0 rgba(0, 0, 0, 0.04); + box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.12), 0 0 6px 0 rgba(0, 0, 0, 0.04); +} + +.el-tabs--border-card > .el-tabs__content { + padding: 15px; +} + +.el-tabs--border-card > .el-tabs__header { + background-color: #f5f7fa; + border-bottom: 1px solid #e4e7ed; + margin: 0; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item { + -webkit-transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + border: 1px solid transparent; + margin-top: -1px; + color: #909399; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item + .el-tabs__item, +.el-tabs--border-card > .el-tabs__header .el-tabs__item:first-child { + margin-left: -1px; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-active { + color: #00a5a7; + background-color: #fff; + border-right-color: #dcdfe6; + border-left-color: #dcdfe6; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item:not(.is-disabled):hover { + color: #00a5a7; +} + +.el-tabs--border-card > .el-tabs__header .el-tabs__item.is-disabled { + color: #c0c4cc; +} + +.el-tabs--border-card > .el-tabs__header .is-scrollable .el-tabs__item:first-child { + margin-left: 0; +} + +.el-tabs--bottom .el-tabs__item.is-bottom:nth-child(2), +.el-tabs--bottom .el-tabs__item.is-top:nth-child(2), +.el-tabs--top .el-tabs__item.is-bottom:nth-child(2), +.el-tabs--top .el-tabs__item.is-top:nth-child(2) { + padding-left: 0; +} + +.el-tabs--bottom .el-tabs__item.is-bottom:last-child, +.el-tabs--bottom .el-tabs__item.is-top:last-child, +.el-tabs--top .el-tabs__item.is-bottom:last-child, +.el-tabs--top .el-tabs__item.is-top:last-child { + padding-right: 0; +} + +.el-tabs--bottom .el-tabs--left > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--bottom .el-tabs--right > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--bottom.el-tabs--border-card > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--bottom.el-tabs--card > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--top .el-tabs--left > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--top .el-tabs--right > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--top.el-tabs--border-card > .el-tabs__header .el-tabs__item:nth-child(2), +.el-tabs--top.el-tabs--card > .el-tabs__header .el-tabs__item:nth-child(2) { + padding-left: 20px; +} + +.el-tabs--bottom .el-tabs--left > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--bottom .el-tabs--right > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--bottom.el-tabs--border-card > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--bottom.el-tabs--card > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--top .el-tabs--left > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--top .el-tabs--right > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--top.el-tabs--border-card > .el-tabs__header .el-tabs__item:last-child, +.el-tabs--top.el-tabs--card > .el-tabs__header .el-tabs__item:last-child { + padding-right: 20px; +} + +.el-tabs--bottom .el-tabs__header.is-bottom { + margin-bottom: 0; + margin-top: 10px; +} + +.el-tabs--bottom.el-tabs--border-card .el-tabs__header.is-bottom { + border-bottom: 0; + border-top: 1px solid #dcdfe6; +} + +.el-tabs--bottom.el-tabs--border-card .el-tabs__nav-wrap.is-bottom { + margin-top: -1px; + margin-bottom: 0; +} + +.el-tabs--bottom.el-tabs--border-card .el-tabs__item.is-bottom:not(.is-active) { + border: 1px solid transparent; +} + +.el-tabs--bottom.el-tabs--border-card .el-tabs__item.is-bottom { + margin: 0 -1px -1px; +} + +.el-tabs--left, +.el-tabs--right { + overflow: hidden; +} + +.el-tabs--left .el-tabs__header.is-left, +.el-tabs--left .el-tabs__header.is-right, +.el-tabs--left .el-tabs__nav-scroll, +.el-tabs--left .el-tabs__nav-wrap.is-left, +.el-tabs--left .el-tabs__nav-wrap.is-right, +.el-tabs--right .el-tabs__header.is-left, +.el-tabs--right .el-tabs__header.is-right, +.el-tabs--right .el-tabs__nav-scroll, +.el-tabs--right .el-tabs__nav-wrap.is-left, +.el-tabs--right .el-tabs__nav-wrap.is-right { + height: 100%; +} + +.el-tabs--left .el-tabs__active-bar.is-left, +.el-tabs--left .el-tabs__active-bar.is-right, +.el-tabs--right .el-tabs__active-bar.is-left, +.el-tabs--right .el-tabs__active-bar.is-right { + top: 0; + bottom: auto; + width: 2px; + height: auto; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left, +.el-tabs--left .el-tabs__nav-wrap.is-right, +.el-tabs--right .el-tabs__nav-wrap.is-left, +.el-tabs--right .el-tabs__nav-wrap.is-right { + margin-bottom: 0; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-next, +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-next, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-next, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-next, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev { + height: 30px; + line-height: 30px; + width: 100%; + text-align: center; + cursor: pointer; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-next i, +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev i, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-next i, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev i, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-next i, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev i, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-next i, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev i { + -webkit-transform: rotateZ(90deg); + transform: rotateZ(90deg); +} + +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-prev, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-prev { + left: auto; + top: 0; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left > .el-tabs__nav-next, +.el-tabs--left .el-tabs__nav-wrap.is-right > .el-tabs__nav-next, +.el-tabs--right .el-tabs__nav-wrap.is-left > .el-tabs__nav-next, +.el-tabs--right .el-tabs__nav-wrap.is-right > .el-tabs__nav-next { + right: auto; + bottom: 0; +} + +.el-tabs--left .el-tabs__active-bar.is-left, +.el-tabs--left .el-tabs__nav-wrap.is-left::after { + right: 0; + left: auto; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left.is-scrollable, +.el-tabs--left .el-tabs__nav-wrap.is-right.is-scrollable, +.el-tabs--right .el-tabs__nav-wrap.is-left.is-scrollable, +.el-tabs--right .el-tabs__nav-wrap.is-right.is-scrollable { + padding: 30px 0; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left::after, +.el-tabs--left .el-tabs__nav-wrap.is-right::after, +.el-tabs--right .el-tabs__nav-wrap.is-left::after, +.el-tabs--right .el-tabs__nav-wrap.is-right::after { + height: 100%; + width: 2px; + bottom: auto; + top: 0; +} + +.el-tabs--left .el-tabs__nav.is-left, +.el-tabs--left .el-tabs__nav.is-right, +.el-tabs--right .el-tabs__nav.is-left, +.el-tabs--right .el-tabs__nav.is-right { + float: none; +} + +.el-tabs--left .el-tabs__item.is-left, +.el-tabs--left .el-tabs__item.is-right, +.el-tabs--right .el-tabs__item.is-left, +.el-tabs--right .el-tabs__item.is-right { + display: block; +} + +.el-tabs--left.el-tabs--card .el-tabs__active-bar.is-left, +.el-tabs--right.el-tabs--card .el-tabs__active-bar.is-right { + display: none; +} + +.el-tabs--left .el-tabs__header.is-left { + float: left; + margin-bottom: 0; + margin-right: 10px; +} + +.el-tabs--left .el-tabs__nav-wrap.is-left { + margin-right: -1px; +} + +.el-tabs--left .el-tabs__item.is-left { + text-align: right; +} + +.el-tabs--left.el-tabs--card .el-tabs__item.is-left { + border-left: none; + border-right: 1px solid #e4e7ed; + border-bottom: none; + border-top: 1px solid #e4e7ed; + text-align: left; +} + +.el-tabs--left.el-tabs--card .el-tabs__item.is-left:first-child { + border-right: 1px solid #e4e7ed; + border-top: none; +} + +.el-tabs--left.el-tabs--card .el-tabs__item.is-left.is-active { + border: 1px solid #e4e7ed; + border-right-color: #fff; + border-left: none; + border-bottom: none; +} + +.el-tabs--left.el-tabs--card .el-tabs__item.is-left.is-active:first-child { + border-top: none; +} + +.el-tabs--left.el-tabs--card .el-tabs__item.is-left.is-active:last-child { + border-bottom: none; +} + +.el-tabs--left.el-tabs--card .el-tabs__nav { + border-radius: 4px 0 0 4px; + border-bottom: 1px solid #e4e7ed; + border-right: none; +} + +.el-tabs--left.el-tabs--card .el-tabs__new-tab { + float: none; +} + +.el-tabs--left.el-tabs--border-card .el-tabs__header.is-left { + border-right: 1px solid #dfe4ed; +} + +.el-tabs--left.el-tabs--border-card .el-tabs__item.is-left { + border: 1px solid transparent; + margin: -1px 0 -1px -1px; +} + +.el-tabs--left.el-tabs--border-card .el-tabs__item.is-left.is-active { + border-color: #d1dbe5 transparent; +} + +.el-tabs--right .el-tabs__header.is-right { + float: right; + margin-bottom: 0; + margin-left: 10px; +} + +.el-tabs--right .el-tabs__nav-wrap.is-right { + margin-left: -1px; +} + +.el-tabs--right .el-tabs__nav-wrap.is-right::after { + left: 0; + right: auto; +} + +.el-tabs--right .el-tabs__active-bar.is-right { + left: 0; +} + +.el-tabs--right.el-tabs--card .el-tabs__item.is-right { + border-bottom: none; + border-top: 1px solid #e4e7ed; +} + +.el-tabs--right.el-tabs--card .el-tabs__item.is-right:first-child { + border-left: 1px solid #e4e7ed; + border-top: none; +} + +.el-tabs--right.el-tabs--card .el-tabs__item.is-right.is-active { + border: 1px solid #e4e7ed; + border-left-color: #fff; + border-right: none; + border-bottom: none; +} + +.el-tabs--right.el-tabs--card .el-tabs__item.is-right.is-active:first-child { + border-top: none; +} + +.el-tabs--right.el-tabs--card .el-tabs__item.is-right.is-active:last-child { + border-bottom: none; +} + +.el-tabs--right.el-tabs--card .el-tabs__nav { + border-radius: 0 4px 4px 0; + border-bottom: 1px solid #e4e7ed; + border-left: none; +} + +.el-tabs--right.el-tabs--border-card .el-tabs__header.is-right { + border-left: 1px solid #dfe4ed; +} + +.el-tabs--right.el-tabs--border-card .el-tabs__item.is-right { + border: 1px solid transparent; + margin: -1px -1px -1px 0; +} + +.el-tabs--right.el-tabs--border-card .el-tabs__item.is-right.is-active { + border-color: #d1dbe5 transparent; +} + +.slideInLeft-transition, +.slideInRight-transition { + display: inline-block; +} + +.slideInRight-enter { + -webkit-animation: slideInRight-enter 0.3s; + animation: slideInRight-enter 0.3s; +} + +.slideInRight-leave { + position: absolute; + left: 0; + right: 0; + -webkit-animation: slideInRight-leave 0.3s; + animation: slideInRight-leave 0.3s; +} + +.slideInLeft-enter { + -webkit-animation: slideInLeft-enter 0.3s; + animation: slideInLeft-enter 0.3s; +} + +.slideInLeft-leave { + position: absolute; + left: 0; + right: 0; + -webkit-animation: slideInLeft-leave 0.3s; + animation: slideInLeft-leave 0.3s; +} + +@-webkit-keyframes slideInRight-enter { + 0% { + opacity: 0; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(100%); + transform: translateX(100%); + } + + to { + opacity: 1; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + } +} + +@keyframes slideInRight-enter { + 0% { + opacity: 0; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(100%); + transform: translateX(100%); + } + + to { + opacity: 1; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + } +} + +@-webkit-keyframes slideInRight-leave { + 0% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + opacity: 1; + } + + 100% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(100%); + transform: translateX(100%); + opacity: 0; + } +} + +@keyframes slideInRight-leave { + 0% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + opacity: 1; + } + + 100% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(100%); + transform: translateX(100%); + opacity: 0; + } +} + +@-webkit-keyframes slideInLeft-enter { + 0% { + opacity: 0; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(-100%); + transform: translateX(-100%); + } + + to { + opacity: 1; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + } +} + +@keyframes slideInLeft-enter { + 0% { + opacity: 0; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(-100%); + transform: translateX(-100%); + } + + to { + opacity: 1; + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + } +} + +@-webkit-keyframes slideInLeft-leave { + 0% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + opacity: 1; + } + + 100% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(-100%); + transform: translateX(-100%); + opacity: 0; + } +} + +@keyframes slideInLeft-leave { + 0% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(0); + transform: translateX(0); + opacity: 1; + } + + 100% { + -webkit-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-transform: translateX(-100%); + transform: translateX(-100%); + opacity: 0; + } +} + +.el-tree { + position: relative; + cursor: default; + background: #fff; + color: #606266; +} + +.el-tree__empty-block { + position: relative; + min-height: 60px; + text-align: center; + width: 100%; + height: 100%; +} + +.el-tree__empty-text { + position: absolute; + left: 50%; + top: 50%; + -webkit-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + color: #909399; +} + +.el-tree__drop-indicator { + position: absolute; + left: 0; + right: 0; + height: 1px; + background-color: #00a5a7; +} + +.el-tree-node { + white-space: nowrap; + outline: 0; +} + +.el-tree-node:focus > .el-tree-node__content { + background-color: #f5f7fa; +} + +.el-tree-node.is-drop-inner > .el-tree-node__content .el-tree-node__label { + background-color: #00a5a7; + color: #fff; +} + +.el-tree-node__content { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + height: 26px; + cursor: pointer; +} + +.el-tree-node__content > .el-tree-node__expand-icon { + padding: 6px; +} + +.el-tree-node__content > label.el-checkbox { + margin-right: 8px; +} + +.el-tree-node__content:hover { + background-color: #f5f7fa; +} + +.el-tree.is-dragging .el-tree-node__content { + cursor: move; +} + +.el-tree.is-dragging.is-drop-not-allow .el-tree-node__content { + cursor: not-allowed; +} + +.el-tree-node__expand-icon { + cursor: pointer; + color: #c0c4cc; + font-size: 12px; + -webkit-transform: rotate(0); + transform: rotate(0); + -webkit-transition: -webkit-transform 0.3s ease-in-out; + transition: -webkit-transform 0.3s ease-in-out; + transition: transform 0.3s ease-in-out; + transition: transform 0.3s ease-in-out, -webkit-transform 0.3s ease-in-out; +} + +.el-tree-node__expand-icon.expanded { + -webkit-transform: rotate(90deg); + transform: rotate(90deg); +} + +.el-tree-node__expand-icon.is-leaf { + color: transparent; + cursor: default; +} + +.el-tree-node__label { + font-size: 14px; +} + +.el-tree-node__loading-icon { + margin-right: 8px; + font-size: 14px; + color: #c0c4cc; +} + +.el-tree-node > .el-tree-node__children { + overflow: hidden; + background-color: transparent; +} + +.el-tree-node.is-expanded > .el-tree-node__children { + display: block; +} + +.el-tree--highlight-current .el-tree-node.is-current > .el-tree-node__content { + background-color: #f0f7ff; +} + +.el-alert { + width: 100%; + padding: 8px 16px; + margin: 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-radius: 4px; + position: relative; + background-color: #fff; + overflow: hidden; + opacity: 1; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-transition: opacity 0.2s; + transition: opacity 0.2s; +} + +.el-alert.is-light .el-alert__closebtn { + color: #c0c4cc; +} + +.el-alert.is-dark .el-alert__closebtn, +.el-alert.is-dark .el-alert__description { + color: #fff; +} + +.el-alert.is-center { + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; +} + +.el-alert--success.is-light { + background-color: #f0f9eb; + color: #67c23a; +} + +.el-alert--success.is-light .el-alert__description { + color: #67c23a; +} + +.el-alert--success.is-dark { + background-color: #67c23a; + color: #fff; +} + +.el-alert--info.is-light { + background-color: #f4f4f5; + color: #909399; +} + +.el-alert--info.is-dark { + background-color: #909399; + color: #fff; +} + +.el-alert--info .el-alert__description { + color: #909399; +} + +.el-alert--warning.is-light { + background-color: #fdf6ec; + color: #e6a23c; +} + +.el-alert--warning.is-light .el-alert__description { + color: #e6a23c; +} + +.el-alert--warning.is-dark { + background-color: #e6a23c; + color: #fff; +} + +.el-alert--error.is-light { + background-color: #fef0f0; + color: #f56c6c; +} + +.el-alert--error.is-light .el-alert__description { + color: #f56c6c; +} + +.el-alert--error.is-dark { + background-color: #f56c6c; + color: #fff; +} + +.el-alert__content { + display: table-cell; + padding: 0 8px; +} + +.el-alert__icon { + font-size: 16px; + width: 16px; +} + +.el-alert__icon.is-big { + font-size: 28px; + width: 28px; +} + +.el-alert__title { + font-size: 13px; + line-height: 18px; +} + +.el-alert__title.is-bold { + font-weight: 700; +} + +.el-alert .el-alert__description { + font-size: 12px; + margin: 5px 0 0; +} + +.el-alert__closebtn { + font-size: 12px; + opacity: 1; + position: absolute; + top: 12px; + right: 15px; + cursor: pointer; +} + +.el-alert-fade-enter, +.el-alert-fade-leave-active, +.el-loading-fade-enter, +.el-loading-fade-leave-active, +.el-notification-fade-leave-active { + opacity: 0; +} + +.el-alert__closebtn.is-customed { + font-style: normal; + font-size: 13px; + top: 9px; +} + +.el-notification { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + width: 330px; + padding: 14px 26px 14px 13px; + border-radius: 8px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border: 1px solid #ebeef5; + position: fixed; + background-color: #fff; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + -webkit-transition: opacity 0.3s, left 0.3s, right 0.3s, top 0.4s, bottom 0.3s, + -webkit-transform 0.3s; + transition: opacity 0.3s, left 0.3s, right 0.3s, top 0.4s, bottom 0.3s, -webkit-transform 0.3s; + transition: opacity 0.3s, transform 0.3s, left 0.3s, right 0.3s, top 0.4s, bottom 0.3s; + transition: opacity 0.3s, transform 0.3s, left 0.3s, right 0.3s, top 0.4s, bottom 0.3s, + -webkit-transform 0.3s; + overflow: hidden; +} + +.el-notification.right { + right: 16px; +} + +.el-notification.left { + left: 16px; +} + +.el-notification__group { + margin-left: 13px; + margin-right: 8px; +} + +.el-notification__title { + font-weight: 700; + font-size: 16px; + color: #303133; + margin: 0; +} + +.el-notification__content { + font-size: 14px; + line-height: 21px; + margin: 6px 0 0; + color: #606266; + text-align: justify; +} + +.el-notification__content p { + margin: 0; +} + +.el-notification__icon { + height: 24px; + width: 24px; + font-size: 24px; +} + +.el-notification__closeBtn { + position: absolute; + top: 18px; + right: 15px; + cursor: pointer; + color: #909399; + font-size: 16px; +} + +.el-notification__closeBtn:hover { + color: #606266; +} + +.el-notification .el-icon-success { + color: #67c23a; +} + +.el-notification .el-icon-error { + color: #f56c6c; +} + +.el-notification .el-icon-info { + color: #909399; +} + +.el-notification .el-icon-warning { + color: #e6a23c; +} + +.el-notification-fade-enter.right { + right: 0; + -webkit-transform: translateX(100%); + transform: translateX(100%); +} + +.el-notification-fade-enter.left { + left: 0; + -webkit-transform: translateX(-100%); + transform: translateX(-100%); +} + +.el-input-number { + position: relative; + display: inline-block; + width: 180px; + line-height: 38px; +} + +.el-input-number .el-input { + display: block; +} + +.el-input-number .el-input__inner { + -webkit-appearance: none; + padding-left: 50px; + padding-right: 50px; + text-align: center; +} + +.el-input-number__decrease, +.el-input-number__increase { + position: absolute; + z-index: 1; + top: 1px; + width: 40px; + height: auto; + text-align: center; + background: #f5f7fa; + color: #606266; + cursor: pointer; + font-size: 13px; +} + +.el-input-number__decrease:hover, +.el-input-number__increase:hover { + color: #00a5a7; +} + +.el-input-number__decrease:hover:not(.is-disabled) + ~ .el-input + .el-input__inner:not(.is-disabled), +.el-input-number__increase:hover:not(.is-disabled) + ~ .el-input + .el-input__inner:not(.is-disabled) { + border-color: #00a5a7; +} + +.el-input-number__decrease.is-disabled, +.el-input-number__increase.is-disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-input-number__increase { + right: 1px; + border-radius: 0 4px 4px 0; + border-left: 1px solid #dcdfe6; +} + +.el-input-number__decrease { + left: 1px; + border-radius: 4px 0 0 4px; + border-right: 1px solid #dcdfe6; +} + +.el-input-number.is-disabled .el-input-number__decrease, +.el-input-number.is-disabled .el-input-number__increase { + border-color: #e4e7ed; + color: #e4e7ed; +} + +.el-input-number.is-disabled .el-input-number__decrease:hover, +.el-input-number.is-disabled .el-input-number__increase:hover { + color: #e4e7ed; + cursor: not-allowed; +} + +.el-input-number--medium { + width: 200px; + line-height: 34px; +} + +.el-input-number--medium .el-input-number__decrease, +.el-input-number--medium .el-input-number__increase { + width: 36px; + font-size: 14px; +} + +.el-input-number--medium .el-input__inner { + padding-left: 43px; + padding-right: 43px; +} + +.el-input-number--small { + width: 130px; + line-height: 30px; +} + +.el-input-number--small .el-input-number__decrease, +.el-input-number--small .el-input-number__increase { + width: 32px; + font-size: 13px; +} + +.el-input-number--small .el-input-number__decrease [class*='el-icon'], +.el-input-number--small .el-input-number__increase [class*='el-icon'] { + -webkit-transform: scale(0.9); + transform: scale(0.9); +} + +.el-input-number--small .el-input__inner { + padding-left: 39px; + padding-right: 39px; +} + +.el-input-number--mini { + width: 130px; + line-height: 26px; +} + +.el-input-number--mini .el-input-number__decrease, +.el-input-number--mini .el-input-number__increase { + width: 28px; + font-size: 12px; +} + +.el-input-number--mini .el-input-number__decrease [class*='el-icon'], +.el-input-number--mini .el-input-number__increase [class*='el-icon'] { + -webkit-transform: scale(0.8); + transform: scale(0.8); +} + +.el-input-number--mini .el-input__inner { + padding-left: 35px; + padding-right: 35px; +} + +.el-input-number.is-without-controls .el-input__inner { + padding-left: 15px; + padding-right: 15px; +} + +.el-input-number.is-controls-right .el-input__inner { + padding-left: 15px; + padding-right: 50px; +} + +.el-input-number.is-controls-right .el-input-number__decrease, +.el-input-number.is-controls-right .el-input-number__increase { + height: auto; + line-height: 19px; +} + +.el-input-number.is-controls-right .el-input-number__decrease [class*='el-icon'], +.el-input-number.is-controls-right .el-input-number__increase [class*='el-icon'] { + -webkit-transform: scale(0.8); + transform: scale(0.8); +} + +.el-input-number.is-controls-right .el-input-number__increase { + border-radius: 0 4px 0 0; + border-bottom: 1px solid #dcdfe6; +} + +.el-input-number.is-controls-right .el-input-number__decrease { + right: 1px; + bottom: 1px; + top: auto; + left: auto; + border-right: none; + border-left: 1px solid #dcdfe6; + border-radius: 0 0 4px; +} + +.el-input-number.is-controls-right[class*='medium'] [class*='decrease'], +.el-input-number.is-controls-right[class*='medium'] [class*='increase'] { + line-height: 17px; +} + +.el-input-number.is-controls-right[class*='small'] [class*='decrease'], +.el-input-number.is-controls-right[class*='small'] [class*='increase'] { + line-height: 15px; +} + +.el-input-number.is-controls-right[class*='mini'] [class*='decrease'], +.el-input-number.is-controls-right[class*='mini'] [class*='increase'] { + line-height: 13px; +} + +.el-tooltip__popper { + position: absolute; + border-radius: 4px; + padding: 10px; + z-index: 2000; + font-size: 12px; + line-height: 1.2; + min-width: 10px; + max-width:600px; + word-wrap: break-word; + word-break: break-all; +} + +.el-tooltip__popper .popper__arrow, +.el-tooltip__popper .popper__arrow::after { + position: absolute; + display: block; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; +} + +.el-tooltip__popper .popper__arrow { + border-width: 6px; +} + +.el-tooltip__popper .popper__arrow::after { + content: ' '; + border-width: 5px; +} + +.el-progress-bar__inner::after, +.el-row::after, +.el-row::before, +.el-slider::after, +.el-slider::before, +.el-slider__button-wrapper::after, +.el-upload-cover::after { + content: ''; +} + +.el-tooltip__popper[x-placement^='top'] { + margin-bottom: 12px; +} + +.el-tooltip__popper[x-placement^='top'] .popper__arrow { + bottom: -6px; + border-top-color: #303133; + border-bottom-width: 0; +} + +.el-tooltip__popper[x-placement^='top'] .popper__arrow::after { + bottom: 1px; + margin-left: -5px; + border-top-color: #303133; + border-bottom-width: 0; +} + +.el-tooltip__popper[x-placement^='bottom'] { + margin-top: 12px; +} + +.el-tooltip__popper[x-placement^='bottom'] .popper__arrow { + top: -6px; + border-top-width: 0; + border-bottom-color: #303133; +} + +.el-tooltip__popper[x-placement^='bottom'] .popper__arrow::after { + top: 1px; + margin-left: -5px; + border-top-width: 0; + border-bottom-color: #303133; +} + +.el-tooltip__popper[x-placement^='right'] { + margin-left: 12px; +} + +.el-tooltip__popper[x-placement^='right'] .popper__arrow { + left: -6px; + border-right-color: #303133; + border-left-width: 0; +} + +.el-tooltip__popper[x-placement^='right'] .popper__arrow::after { + bottom: -5px; + left: 1px; + border-right-color: #303133; + border-left-width: 0; +} + +.el-tooltip__popper[x-placement^='left'] { + margin-right: 12px; +} + +.el-tooltip__popper[x-placement^='left'] .popper__arrow { + right: -6px; + border-right-width: 0; + border-left-color: #303133; +} + +.el-tooltip__popper[x-placement^='left'] .popper__arrow::after { + right: 1px; + bottom: -5px; + margin-left: -5px; + border-right-width: 0; + border-left-color: #303133; +} + +.el-tooltip__popper.is-dark { + background: #303133; + color: #fff; +} + +.el-tooltip__popper.is-light { + background: #fff; + border: 1px solid #303133; +} + +.el-tooltip__popper.is-light[x-placement^='top'] .popper__arrow { + border-top-color: #303133; +} + +.el-tooltip__popper.is-light[x-placement^='top'] .popper__arrow::after { + border-top-color: #fff; +} + +.el-tooltip__popper.is-light[x-placement^='bottom'] .popper__arrow { + border-bottom-color: #303133; +} + +.el-tooltip__popper.is-light[x-placement^='bottom'] .popper__arrow::after { + border-bottom-color: #fff; +} + +.el-tooltip__popper.is-light[x-placement^='left'] .popper__arrow { + border-left-color: #303133; +} + +.el-tooltip__popper.is-light[x-placement^='left'] .popper__arrow::after { + border-left-color: #fff; +} + +.el-tooltip__popper.is-light[x-placement^='right'] .popper__arrow { + border-right-color: #303133; +} + +.el-tooltip__popper.is-light[x-placement^='right'] .popper__arrow::after { + border-right-color: #fff; +} + +.el-slider::after, +.el-slider::before { + display: table; +} + +.el-slider__button-wrapper .el-tooltip, +.el-slider__button-wrapper::after { + vertical-align: middle; + display: inline-block; +} + +.el-slider::after { + clear: both; +} + +.el-slider__runway { + width: 100%; + height: 6px; + margin: 16px 0; + background-color: #e4e7ed; + border-radius: 3px; + position: relative; + cursor: pointer; + vertical-align: middle; +} + +.el-slider__runway.show-input { + margin-right: 160px; + width: auto; +} + +.el-slider__runway.disabled { + cursor: default; +} + +.el-slider__runway.disabled .el-slider__bar { + background-color: #c0c4cc; +} + +.el-slider__runway.disabled .el-slider__button { + border-color: #c0c4cc; +} + +.el-slider__runway.disabled .el-slider__button-wrapper.dragging, +.el-slider__runway.disabled .el-slider__button-wrapper.hover, +.el-slider__runway.disabled .el-slider__button-wrapper:hover { + cursor: not-allowed; +} + +.el-slider__runway.disabled .el-slider__button.dragging, +.el-slider__runway.disabled .el-slider__button.hover, +.el-slider__runway.disabled .el-slider__button:hover { + -webkit-transform: scale(1); + transform: scale(1); + cursor: not-allowed; +} + +.el-slider__button-wrapper, +.el-slider__stop { + -webkit-transform: translateX(-50%); + position: absolute; +} + +.el-slider__input { + float: right; + margin-top: 3px; + width: 130px; +} + +.el-slider__input.el-input-number--mini { + margin-top: 5px; +} + +.el-slider__input.el-input-number--medium { + margin-top: 0; +} + +.el-slider__input.el-input-number--large { + margin-top: -2px; +} + +.el-slider__bar { + height: 6px; + background-color: #00a5a7; + border-top-left-radius: 3px; + border-bottom-left-radius: 3px; + position: absolute; +} + +.el-slider__button-wrapper { + height: 36px; + width: 36px; + z-index: 1001; + top: -15px; + transform: translateX(-50%); + background-color: transparent; + text-align: center; + -moz-user-select: none; + user-select: none; + line-height: normal; +} + +.el-slider__button, +.el-step__icon-inner { + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; +} + +.el-slider__button-wrapper::after { + height: 100%; +} + +.el-slider__button-wrapper.hover, +.el-slider__button-wrapper:hover { + cursor: -webkit-grab; + cursor: grab; +} + +.el-slider__button-wrapper.dragging { + cursor: -webkit-grabbing; + cursor: grabbing; +} + +.el-slider__button { + width: 16px; + height: 16px; + border: 2px solid #00a5a7; + background-color: #fff; + border-radius: 50%; + -webkit-transition: 0.2s; + transition: 0.2s; + user-select: none; +} + +.el-slider__button.dragging, +.el-slider__button.hover, +.el-slider__button:hover { + -webkit-transform: scale(1.2); + transform: scale(1.2); +} + +.el-slider__button.hover, +.el-slider__button:hover { + cursor: -webkit-grab; + cursor: grab; +} + +.el-slider__button.dragging { + cursor: -webkit-grabbing; + cursor: grabbing; +} + +.el-slider__stop { + height: 6px; + width: 6px; + border-radius: 100%; + background-color: #fff; + transform: translateX(-50%); +} + +.el-slider__marks { + top: 0; + left: 12px; + width: 18px; + height: 100%; +} + +.el-slider__marks-text { + position: absolute; + -webkit-transform: translateX(-50%); + transform: translateX(-50%); + font-size: 14px; + color: #909399; + margin-top: 15px; +} + +.el-slider.is-vertical { + position: relative; +} + +.el-slider.is-vertical .el-slider__runway { + width: 6px; + height: 100%; + margin: 0 16px; +} + +.el-slider.is-vertical .el-slider__bar { + width: 6px; + height: auto; + border-radius: 0 0 3px 3px; +} + +.el-slider.is-vertical .el-slider__button-wrapper { + top: auto; + left: -15px; + -webkit-transform: translateY(50%); + transform: translateY(50%); +} + +.el-slider.is-vertical .el-slider__stop { + -webkit-transform: translateY(50%); + transform: translateY(50%); +} + +.el-slider.is-vertical.el-slider--with-input { + padding-bottom: 58px; +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input { + overflow: visible; + float: none; + position: absolute; + bottom: 22px; + width: 36px; + margin-top: 15px; +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input .el-input__inner { + text-align: center; + padding-left: 5px; + padding-right: 5px; +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input .el-input-number__decrease, +.el-slider.is-vertical.el-slider--with-input .el-slider__input .el-input-number__increase { + top: 32px; + margin-top: -1px; + border: 1px solid #dcdfe6; + line-height: 20px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + -webkit-transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input .el-input-number__decrease { + width: 18px; + right: 18px; + border-bottom-left-radius: 4px; +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input .el-input-number__increase { + width: 19px; + border-bottom-right-radius: 4px; +} + +.el-slider.is-vertical.el-slider--with-input + .el-slider__input + .el-input-number__increase + ~ .el-input + .el-input__inner { + border-bottom-left-radius: 0; + border-bottom-right-radius: 0; +} + +.el-slider.is-vertical.el-slider--with-input .el-slider__input:hover .el-input-number__decrease, +.el-slider.is-vertical.el-slider--with-input + .el-slider__input:hover + .el-input-number__increase { + border-color: #c0c4cc; +} + +.el-slider.is-vertical.el-slider--with-input + .el-slider__input:active + .el-input-number__decrease, +.el-slider.is-vertical.el-slider--with-input + .el-slider__input:active + .el-input-number__increase { + border-color: #00a5a7; +} + +.el-slider.is-vertical .el-slider__marks-text { + margin-top: 0; + left: 15px; + -webkit-transform: translateY(50%); + transform: translateY(50%); +} + +.el-loading-parent--relative { + position: relative !important; +} + +.el-loading-parent--hidden { + overflow: hidden !important; +} + +.el-loading-mask { + position: absolute; + z-index: 2000; + background-color: rgba(255, 255, 255, 0.9); + margin: 0; + top: 0; + right: 0; + bottom: 0; + left: 0; + -webkit-transition: opacity 0.3s; + transition: opacity 0.3s; +} + +.el-loading-mask.is-fullscreen { + position: fixed; +} + +.el-loading-mask.is-fullscreen .el-loading-spinner { + margin-top: -25px; +} + +.el-loading-mask.is-fullscreen .el-loading-spinner .circular { + height: 50px; + width: 50px; +} + +.el-loading-spinner { + top: 50%; + margin-top: -21px; + width: 100%; + text-align: center; + position: absolute; +} + +.el-col-pull-0, +.el-col-pull-1, +.el-col-pull-10, +.el-col-pull-11, +.el-col-pull-13, +.el-col-pull-14, +.el-col-pull-15, +.el-col-pull-16, +.el-col-pull-17, +.el-col-pull-18, +.el-col-pull-19, +.el-col-pull-2, +.el-col-pull-20, +.el-col-pull-21, +.el-col-pull-22, +.el-col-pull-23, +.el-col-pull-24, +.el-col-pull-3, +.el-col-pull-4, +.el-col-pull-5, +.el-col-pull-6, +.el-col-pull-7, +.el-col-pull-8, +.el-col-pull-9, +.el-col-push-0, +.el-col-push-1, +.el-col-push-10, +.el-col-push-11, +.el-col-push-12, +.el-col-push-13, +.el-col-push-14, +.el-col-push-15, +.el-col-push-16, +.el-col-push-17, +.el-col-push-18, +.el-col-push-19, +.el-col-push-2, +.el-col-push-20, +.el-col-push-21, +.el-col-push-22, +.el-col-push-23, +.el-col-push-24, +.el-col-push-3, +.el-col-push-4, +.el-col-push-5, +.el-col-push-6, +.el-col-push-7, +.el-col-push-8, +.el-col-push-9, +.el-row { + position: relative; +} + +.el-loading-spinner .el-loading-text { + color: #ffffff; + margin: 3px 0; + font-size: 18px; +} + +.el-loading-spinner .circular { + height: 42px; + width: 42px; + -webkit-animation: loading-rotate 2s linear infinite; + animation: loading-rotate 2s linear infinite; +} + +.el-loading-spinner .path { + -webkit-animation: loading-dash 1.5s ease-in-out infinite; + animation: loading-dash 1.5s ease-in-out infinite; + stroke-dasharray: 90, 150; + stroke-dashoffset: 0; + stroke-width: 5; + stroke: #ffffff; + stroke-linecap: round; +} + +.el-loading-spinner i { + stroke: #ffffff; +} + +@-webkit-keyframes loading-rotate { + 100% { + -webkit-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +@keyframes loading-rotate { + 100% { + -webkit-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +@-webkit-keyframes loading-dash { + 0% { + stroke-dasharray: 1, 200; + stroke-dashoffset: 0; + } + + 50% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -40px; + } + + 100% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -120px; + } +} + +@keyframes loading-dash { + 0% { + stroke-dasharray: 1, 200; + stroke-dashoffset: 0; + } + + 50% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -40px; + } + + 100% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -120px; + } +} + +.el-row { + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-row::after, +.el-row::before { + display: table; +} + +.el-row::after { + clear: both; +} + +.el-row--flex { + display: -webkit-box; + display: -ms-flexbox; + display: flex; +} + +.el-col-0, +.el-row--flex:after, +.el-row--flex:before { + display: none; +} + +.el-row--flex.is-justify-center { + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; +} + +.el-row--flex.is-justify-end { + -webkit-box-pack: end; + -ms-flex-pack: end; + justify-content: flex-end; +} + +.el-row--flex.is-justify-space-between { + -webkit-box-pack: justify; + -ms-flex-pack: justify; + justify-content: space-between; +} + +.el-row--flex.is-justify-space-around { + -ms-flex-pack: distribute; + justify-content: space-around; +} + +.el-row--flex.is-align-middle { + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-row--flex.is-align-bottom { + -webkit-box-align: end; + -ms-flex-align: end; + align-items: flex-end; +} + +[class*='el-col-'] { + float: left; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-upload--picture-card, +.el-upload-dragger { + -webkit-box-sizing: border-box; + cursor: pointer; +} + +.el-col-0 { + width: 0%; +} + +.el-col-offset-0 { + margin-left: 0; +} + +.el-col-pull-0 { + right: 0; +} + +.el-col-push-0 { + left: 0; +} + +.el-col-1 { + width: 4.16667%; +} + +.el-col-offset-1 { + margin-left: 4.16667%; +} + +.el-col-pull-1 { + right: 4.16667%; +} + +.el-col-push-1 { + left: 4.16667%; +} + +.el-col-2 { + width: 8.33333%; +} + +.el-col-offset-2 { + margin-left: 8.33333%; +} + +.el-col-pull-2 { + right: 8.33333%; +} + +.el-col-push-2 { + left: 8.33333%; +} + +.el-col-3 { + width: 12.5%; +} + +.el-col-offset-3 { + margin-left: 12.5%; +} + +.el-col-pull-3 { + right: 12.5%; +} + +.el-col-push-3 { + left: 12.5%; +} + +.el-col-4 { + width: 16.66667%; +} + +.el-col-offset-4 { + margin-left: 16.66667%; +} + +.el-col-pull-4 { + right: 16.66667%; +} + +.el-col-push-4 { + left: 16.66667%; +} + +.el-col-5 { + width: 20.83333%; +} + +.el-col-offset-5 { + margin-left: 20.83333%; +} + +.el-col-pull-5 { + right: 20.83333%; +} + +.el-col-push-5 { + left: 20.83333%; +} + +.el-col-6 { + width: 25%; +} + +.el-col-offset-6 { + margin-left: 25%; +} + +.el-col-pull-6 { + right: 25%; +} + +.el-col-push-6 { + left: 25%; +} + +.el-col-7 { + width: 29.16667%; +} + +.el-col-offset-7 { + margin-left: 29.16667%; +} + +.el-col-pull-7 { + right: 29.16667%; +} + +.el-col-push-7 { + left: 29.16667%; +} + +.el-col-8 { + width: 33.33333%; +} + +.el-col-offset-8 { + margin-left: 33.33333%; +} + +.el-col-pull-8 { + right: 33.33333%; +} + +.el-col-push-8 { + left: 33.33333%; +} + +.el-col-9 { + width: 37.5%; +} + +.el-col-offset-9 { + margin-left: 37.5%; +} + +.el-col-pull-9 { + right: 37.5%; +} + +.el-col-push-9 { + left: 37.5%; +} + +.el-col-10 { + width: 41.66667%; +} + +.el-col-offset-10 { + margin-left: 41.66667%; +} + +.el-col-pull-10 { + right: 41.66667%; +} + +.el-col-push-10 { + left: 41.66667%; +} + +.el-col-11 { + width: 45.83333%; +} + +.el-col-offset-11 { + margin-left: 45.83333%; +} + +.el-col-pull-11 { + right: 45.83333%; +} + +.el-col-push-11 { + left: 45.83333%; +} + +.el-col-12 { + width: 50%; +} + +.el-col-offset-12 { + margin-left: 50%; +} + +.el-col-pull-12 { + position: relative; + right: 50%; +} + +.el-col-push-12 { + left: 50%; +} + +.el-col-13 { + width: 54.16667%; +} + +.el-col-offset-13 { + margin-left: 54.16667%; +} + +.el-col-pull-13 { + right: 54.16667%; +} + +.el-col-push-13 { + left: 54.16667%; +} + +.el-col-14 { + width: 58.33333%; +} + +.el-col-offset-14 { + margin-left: 58.33333%; +} + +.el-col-pull-14 { + right: 58.33333%; +} + +.el-col-push-14 { + left: 58.33333%; +} + +.el-col-15 { + width: 62.5%; +} + +.el-col-offset-15 { + margin-left: 62.5%; +} + +.el-col-pull-15 { + right: 62.5%; +} + +.el-col-push-15 { + left: 62.5%; +} + +.el-col-16 { + width: 66.66667%; +} + +.el-col-offset-16 { + margin-left: 66.66667%; +} + +.el-col-pull-16 { + right: 66.66667%; +} + +.el-col-push-16 { + left: 66.66667%; +} + +.el-col-17 { + width: 70.83333%; +} + +.el-col-offset-17 { + margin-left: 70.83333%; +} + +.el-col-pull-17 { + right: 70.83333%; +} + +.el-col-push-17 { + left: 70.83333%; +} + +.el-col-18 { + width: 75%; +} + +.el-col-offset-18 { + margin-left: 75%; +} + +.el-col-pull-18 { + right: 75%; +} + +.el-col-push-18 { + left: 75%; +} + +.el-col-19 { + width: 79.16667%; +} + +.el-col-offset-19 { + margin-left: 79.16667%; +} + +.el-col-pull-19 { + right: 79.16667%; +} + +.el-col-push-19 { + left: 79.16667%; +} + +.el-col-20 { + width: 83.33333%; +} + +.el-col-offset-20 { + margin-left: 83.33333%; +} + +.el-col-pull-20 { + right: 83.33333%; +} + +.el-col-push-20 { + left: 83.33333%; +} + +.el-col-21 { + width: 87.5%; +} + +.el-col-offset-21 { + margin-left: 87.5%; +} + +.el-col-pull-21 { + right: 87.5%; +} + +.el-col-push-21 { + left: 87.5%; +} + +.el-col-22 { + width: 91.66667%; +} + +.el-col-offset-22 { + margin-left: 91.66667%; +} + +.el-col-pull-22 { + right: 91.66667%; +} + +.el-col-push-22 { + left: 91.66667%; +} + +.el-col-23 { + width: 95.83333%; +} + +.el-col-offset-23 { + margin-left: 95.83333%; +} + +.el-col-pull-23 { + right: 95.83333%; +} + +.el-col-push-23 { + left: 95.83333%; +} + +.el-col-24 { + width: 100%; +} + +.el-col-offset-24 { + margin-left: 100%; +} + +.el-col-pull-24 { + right: 100%; +} + +.el-col-push-24 { + left: 100%; +} + +@media only screen and (max-width: 767px) { + .el-col-xs-0 { + display: none; + width: 0%; + } + + .el-col-xs-offset-0 { + margin-left: 0; + } + + .el-col-xs-pull-0 { + position: relative; + right: 0; + } + + .el-col-xs-push-0 { + position: relative; + left: 0; + } + + .el-col-xs-1 { + width: 4.16667%; + } + + .el-col-xs-offset-1 { + margin-left: 4.16667%; + } + + .el-col-xs-pull-1 { + position: relative; + right: 4.16667%; + } + + .el-col-xs-push-1 { + position: relative; + left: 4.16667%; + } + + .el-col-xs-2 { + width: 8.33333%; + } + + .el-col-xs-offset-2 { + margin-left: 8.33333%; + } + + .el-col-xs-pull-2 { + position: relative; + right: 8.33333%; + } + + .el-col-xs-push-2 { + position: relative; + left: 8.33333%; + } + + .el-col-xs-3 { + width: 12.5%; + } + + .el-col-xs-offset-3 { + margin-left: 12.5%; + } + + .el-col-xs-pull-3 { + position: relative; + right: 12.5%; + } + + .el-col-xs-push-3 { + position: relative; + left: 12.5%; + } + + .el-col-xs-4 { + width: 16.66667%; + } + + .el-col-xs-offset-4 { + margin-left: 16.66667%; + } + + .el-col-xs-pull-4 { + position: relative; + right: 16.66667%; + } + + .el-col-xs-push-4 { + position: relative; + left: 16.66667%; + } + + .el-col-xs-5 { + width: 20.83333%; + } + + .el-col-xs-offset-5 { + margin-left: 20.83333%; + } + + .el-col-xs-pull-5 { + position: relative; + right: 20.83333%; + } + + .el-col-xs-push-5 { + position: relative; + left: 20.83333%; + } + + .el-col-xs-6 { + width: 25%; + } + + .el-col-xs-offset-6 { + margin-left: 25%; + } + + .el-col-xs-pull-6 { + position: relative; + right: 25%; + } + + .el-col-xs-push-6 { + position: relative; + left: 25%; + } + + .el-col-xs-7 { + width: 29.16667%; + } + + .el-col-xs-offset-7 { + margin-left: 29.16667%; + } + + .el-col-xs-pull-7 { + position: relative; + right: 29.16667%; + } + + .el-col-xs-push-7 { + position: relative; + left: 29.16667%; + } + + .el-col-xs-8 { + width: 33.33333%; + } + + .el-col-xs-offset-8 { + margin-left: 33.33333%; + } + + .el-col-xs-pull-8 { + position: relative; + right: 33.33333%; + } + + .el-col-xs-push-8 { + position: relative; + left: 33.33333%; + } + + .el-col-xs-9 { + width: 37.5%; + } + + .el-col-xs-offset-9 { + margin-left: 37.5%; + } + + .el-col-xs-pull-9 { + position: relative; + right: 37.5%; + } + + .el-col-xs-push-9 { + position: relative; + left: 37.5%; + } + + .el-col-xs-10 { + width: 41.66667%; + } + + .el-col-xs-offset-10 { + margin-left: 41.66667%; + } + + .el-col-xs-pull-10 { + position: relative; + right: 41.66667%; + } + + .el-col-xs-push-10 { + position: relative; + left: 41.66667%; + } + + .el-col-xs-11 { + width: 45.83333%; + } + + .el-col-xs-offset-11 { + margin-left: 45.83333%; + } + + .el-col-xs-pull-11 { + position: relative; + right: 45.83333%; + } + + .el-col-xs-push-11 { + position: relative; + left: 45.83333%; + } + + .el-col-xs-12 { + width: 50%; + } + + .el-col-xs-offset-12 { + margin-left: 50%; + } + + .el-col-xs-pull-12 { + position: relative; + right: 50%; + } + + .el-col-xs-push-12 { + position: relative; + left: 50%; + } + + .el-col-xs-13 { + width: 54.16667%; + } + + .el-col-xs-offset-13 { + margin-left: 54.16667%; + } + + .el-col-xs-pull-13 { + position: relative; + right: 54.16667%; + } + + .el-col-xs-push-13 { + position: relative; + left: 54.16667%; + } + + .el-col-xs-14 { + width: 58.33333%; + } + + .el-col-xs-offset-14 { + margin-left: 58.33333%; + } + + .el-col-xs-pull-14 { + position: relative; + right: 58.33333%; + } + + .el-col-xs-push-14 { + position: relative; + left: 58.33333%; + } + + .el-col-xs-15 { + width: 62.5%; + } + + .el-col-xs-offset-15 { + margin-left: 62.5%; + } + + .el-col-xs-pull-15 { + position: relative; + right: 62.5%; + } + + .el-col-xs-push-15 { + position: relative; + left: 62.5%; + } + + .el-col-xs-16 { + width: 66.66667%; + } + + .el-col-xs-offset-16 { + margin-left: 66.66667%; + } + + .el-col-xs-pull-16 { + position: relative; + right: 66.66667%; + } + + .el-col-xs-push-16 { + position: relative; + left: 66.66667%; + } + + .el-col-xs-17 { + width: 70.83333%; + } + + .el-col-xs-offset-17 { + margin-left: 70.83333%; + } + + .el-col-xs-pull-17 { + position: relative; + right: 70.83333%; + } + + .el-col-xs-push-17 { + position: relative; + left: 70.83333%; + } + + .el-col-xs-18 { + width: 75%; + } + + .el-col-xs-offset-18 { + margin-left: 75%; + } + + .el-col-xs-pull-18 { + position: relative; + right: 75%; + } + + .el-col-xs-push-18 { + position: relative; + left: 75%; + } + + .el-col-xs-19 { + width: 79.16667%; + } + + .el-col-xs-offset-19 { + margin-left: 79.16667%; + } + + .el-col-xs-pull-19 { + position: relative; + right: 79.16667%; + } + + .el-col-xs-push-19 { + position: relative; + left: 79.16667%; + } + + .el-col-xs-20 { + width: 83.33333%; + } + + .el-col-xs-offset-20 { + margin-left: 83.33333%; + } + + .el-col-xs-pull-20 { + position: relative; + right: 83.33333%; + } + + .el-col-xs-push-20 { + position: relative; + left: 83.33333%; + } + + .el-col-xs-21 { + width: 87.5%; + } + + .el-col-xs-offset-21 { + margin-left: 87.5%; + } + + .el-col-xs-pull-21 { + position: relative; + right: 87.5%; + } + + .el-col-xs-push-21 { + position: relative; + left: 87.5%; + } + + .el-col-xs-22 { + width: 91.66667%; + } + + .el-col-xs-offset-22 { + margin-left: 91.66667%; + } + + .el-col-xs-pull-22 { + position: relative; + right: 91.66667%; + } + + .el-col-xs-push-22 { + position: relative; + left: 91.66667%; + } + + .el-col-xs-23 { + width: 95.83333%; + } + + .el-col-xs-offset-23 { + margin-left: 95.83333%; + } + + .el-col-xs-pull-23 { + position: relative; + right: 95.83333%; + } + + .el-col-xs-push-23 { + position: relative; + left: 95.83333%; + } + + .el-col-xs-24 { + width: 100%; + } + + .el-col-xs-offset-24 { + margin-left: 100%; + } + + .el-col-xs-pull-24 { + position: relative; + right: 100%; + } + + .el-col-xs-push-24 { + position: relative; + left: 100%; + } +} + +@media only screen and (min-width: 768px) { + .el-col-sm-0 { + display: none; + width: 0%; + } + + .el-col-sm-offset-0 { + margin-left: 0; + } + + .el-col-sm-pull-0 { + position: relative; + right: 0; + } + + .el-col-sm-push-0 { + position: relative; + left: 0; + } + + .el-col-sm-1 { + width: 4.16667%; + } + + .el-col-sm-offset-1 { + margin-left: 4.16667%; + } + + .el-col-sm-pull-1 { + position: relative; + right: 4.16667%; + } + + .el-col-sm-push-1 { + position: relative; + left: 4.16667%; + } + + .el-col-sm-2 { + width: 8.33333%; + } + + .el-col-sm-offset-2 { + margin-left: 8.33333%; + } + + .el-col-sm-pull-2 { + position: relative; + right: 8.33333%; + } + + .el-col-sm-push-2 { + position: relative; + left: 8.33333%; + } + + .el-col-sm-3 { + width: 12.5%; + } + + .el-col-sm-offset-3 { + margin-left: 12.5%; + } + + .el-col-sm-pull-3 { + position: relative; + right: 12.5%; + } + + .el-col-sm-push-3 { + position: relative; + left: 12.5%; + } + + .el-col-sm-4 { + width: 16.66667%; + } + + .el-col-sm-offset-4 { + margin-left: 16.66667%; + } + + .el-col-sm-pull-4 { + position: relative; + right: 16.66667%; + } + + .el-col-sm-push-4 { + position: relative; + left: 16.66667%; + } + + .el-col-sm-5 { + width: 20.83333%; + } + + .el-col-sm-offset-5 { + margin-left: 20.83333%; + } + + .el-col-sm-pull-5 { + position: relative; + right: 20.83333%; + } + + .el-col-sm-push-5 { + position: relative; + left: 20.83333%; + } + + .el-col-sm-6 { + width: 25%; + } + + .el-col-sm-offset-6 { + margin-left: 25%; + } + + .el-col-sm-pull-6 { + position: relative; + right: 25%; + } + + .el-col-sm-push-6 { + position: relative; + left: 25%; + } + + .el-col-sm-7 { + width: 29.16667%; + } + + .el-col-sm-offset-7 { + margin-left: 29.16667%; + } + + .el-col-sm-pull-7 { + position: relative; + right: 29.16667%; + } + + .el-col-sm-push-7 { + position: relative; + left: 29.16667%; + } + + .el-col-sm-8 { + width: 33.33333%; + } + + .el-col-sm-offset-8 { + margin-left: 33.33333%; + } + + .el-col-sm-pull-8 { + position: relative; + right: 33.33333%; + } + + .el-col-sm-push-8 { + position: relative; + left: 33.33333%; + } + + .el-col-sm-9 { + width: 37.5%; + } + + .el-col-sm-offset-9 { + margin-left: 37.5%; + } + + .el-col-sm-pull-9 { + position: relative; + right: 37.5%; + } + + .el-col-sm-push-9 { + position: relative; + left: 37.5%; + } + + .el-col-sm-10 { + width: 41.66667%; + } + + .el-col-sm-offset-10 { + margin-left: 41.66667%; + } + + .el-col-sm-pull-10 { + position: relative; + right: 41.66667%; + } + + .el-col-sm-push-10 { + position: relative; + left: 41.66667%; + } + + .el-col-sm-11 { + width: 45.83333%; + } + + .el-col-sm-offset-11 { + margin-left: 45.83333%; + } + + .el-col-sm-pull-11 { + position: relative; + right: 45.83333%; + } + + .el-col-sm-push-11 { + position: relative; + left: 45.83333%; + } + + .el-col-sm-12 { + width: 50%; + } + + .el-col-sm-offset-12 { + margin-left: 50%; + } + + .el-col-sm-pull-12 { + position: relative; + right: 50%; + } + + .el-col-sm-push-12 { + position: relative; + left: 50%; + } + + .el-col-sm-13 { + width: 54.16667%; + } + + .el-col-sm-offset-13 { + margin-left: 54.16667%; + } + + .el-col-sm-pull-13 { + position: relative; + right: 54.16667%; + } + + .el-col-sm-push-13 { + position: relative; + left: 54.16667%; + } + + .el-col-sm-14 { + width: 58.33333%; + } + + .el-col-sm-offset-14 { + margin-left: 58.33333%; + } + + .el-col-sm-pull-14 { + position: relative; + right: 58.33333%; + } + + .el-col-sm-push-14 { + position: relative; + left: 58.33333%; + } + + .el-col-sm-15 { + width: 62.5%; + } + + .el-col-sm-offset-15 { + margin-left: 62.5%; + } + + .el-col-sm-pull-15 { + position: relative; + right: 62.5%; + } + + .el-col-sm-push-15 { + position: relative; + left: 62.5%; + } + + .el-col-sm-16 { + width: 66.66667%; + } + + .el-col-sm-offset-16 { + margin-left: 66.66667%; + } + + .el-col-sm-pull-16 { + position: relative; + right: 66.66667%; + } + + .el-col-sm-push-16 { + position: relative; + left: 66.66667%; + } + + .el-col-sm-17 { + width: 70.83333%; + } + + .el-col-sm-offset-17 { + margin-left: 70.83333%; + } + + .el-col-sm-pull-17 { + position: relative; + right: 70.83333%; + } + + .el-col-sm-push-17 { + position: relative; + left: 70.83333%; + } + + .el-col-sm-18 { + width: 75%; + } + + .el-col-sm-offset-18 { + margin-left: 75%; + } + + .el-col-sm-pull-18 { + position: relative; + right: 75%; + } + + .el-col-sm-push-18 { + position: relative; + left: 75%; + } + + .el-col-sm-19 { + width: 79.16667%; + } + + .el-col-sm-offset-19 { + margin-left: 79.16667%; + } + + .el-col-sm-pull-19 { + position: relative; + right: 79.16667%; + } + + .el-col-sm-push-19 { + position: relative; + left: 79.16667%; + } + + .el-col-sm-20 { + width: 83.33333%; + } + + .el-col-sm-offset-20 { + margin-left: 83.33333%; + } + + .el-col-sm-pull-20 { + position: relative; + right: 83.33333%; + } + + .el-col-sm-push-20 { + position: relative; + left: 83.33333%; + } + + .el-col-sm-21 { + width: 87.5%; + } + + .el-col-sm-offset-21 { + margin-left: 87.5%; + } + + .el-col-sm-pull-21 { + position: relative; + right: 87.5%; + } + + .el-col-sm-push-21 { + position: relative; + left: 87.5%; + } + + .el-col-sm-22 { + width: 91.66667%; + } + + .el-col-sm-offset-22 { + margin-left: 91.66667%; + } + + .el-col-sm-pull-22 { + position: relative; + right: 91.66667%; + } + + .el-col-sm-push-22 { + position: relative; + left: 91.66667%; + } + + .el-col-sm-23 { + width: 95.83333%; + } + + .el-col-sm-offset-23 { + margin-left: 95.83333%; + } + + .el-col-sm-pull-23 { + position: relative; + right: 95.83333%; + } + + .el-col-sm-push-23 { + position: relative; + left: 95.83333%; + } + + .el-col-sm-24 { + width: 100%; + } + + .el-col-sm-offset-24 { + margin-left: 100%; + } + + .el-col-sm-pull-24 { + position: relative; + right: 100%; + } + + .el-col-sm-push-24 { + position: relative; + left: 100%; + } +} + +@media only screen and (min-width: 992px) { + .el-col-md-0 { + display: none; + width: 0%; + } + + .el-col-md-offset-0 { + margin-left: 0; + } + + .el-col-md-pull-0 { + position: relative; + right: 0; + } + + .el-col-md-push-0 { + position: relative; + left: 0; + } + + .el-col-md-1 { + width: 4.16667%; + } + + .el-col-md-offset-1 { + margin-left: 4.16667%; + } + + .el-col-md-pull-1 { + position: relative; + right: 4.16667%; + } + + .el-col-md-push-1 { + position: relative; + left: 4.16667%; + } + + .el-col-md-2 { + width: 8.33333%; + } + + .el-col-md-offset-2 { + margin-left: 8.33333%; + } + + .el-col-md-pull-2 { + position: relative; + right: 8.33333%; + } + + .el-col-md-push-2 { + position: relative; + left: 8.33333%; + } + + .el-col-md-3 { + width: 12.5%; + } + + .el-col-md-offset-3 { + margin-left: 12.5%; + } + + .el-col-md-pull-3 { + position: relative; + right: 12.5%; + } + + .el-col-md-push-3 { + position: relative; + left: 12.5%; + } + + .el-col-md-4 { + width: 16.66667%; + } + + .el-col-md-offset-4 { + margin-left: 16.66667%; + } + + .el-col-md-pull-4 { + position: relative; + right: 16.66667%; + } + + .el-col-md-push-4 { + position: relative; + left: 16.66667%; + } + + .el-col-md-5 { + width: 20.83333%; + } + + .el-col-md-offset-5 { + margin-left: 20.83333%; + } + + .el-col-md-pull-5 { + position: relative; + right: 20.83333%; + } + + .el-col-md-push-5 { + position: relative; + left: 20.83333%; + } + + .el-col-md-6 { + width: 25%; + } + + .el-col-md-offset-6 { + margin-left: 25%; + } + + .el-col-md-pull-6 { + position: relative; + right: 25%; + } + + .el-col-md-push-6 { + position: relative; + left: 25%; + } + + .el-col-md-7 { + width: 29.16667%; + } + + .el-col-md-offset-7 { + margin-left: 29.16667%; + } + + .el-col-md-pull-7 { + position: relative; + right: 29.16667%; + } + + .el-col-md-push-7 { + position: relative; + left: 29.16667%; + } + + .el-col-md-8 { + width: 33.33333%; + } + + .el-col-md-offset-8 { + margin-left: 33.33333%; + } + + .el-col-md-pull-8 { + position: relative; + right: 33.33333%; + } + + .el-col-md-push-8 { + position: relative; + left: 33.33333%; + } + + .el-col-md-9 { + width: 37.5%; + } + + .el-col-md-offset-9 { + margin-left: 37.5%; + } + + .el-col-md-pull-9 { + position: relative; + right: 37.5%; + } + + .el-col-md-push-9 { + position: relative; + left: 37.5%; + } + + .el-col-md-10 { + width: 41.66667%; + } + + .el-col-md-offset-10 { + margin-left: 41.66667%; + } + + .el-col-md-pull-10 { + position: relative; + right: 41.66667%; + } + + .el-col-md-push-10 { + position: relative; + left: 41.66667%; + } + + .el-col-md-11 { + width: 45.83333%; + } + + .el-col-md-offset-11 { + margin-left: 45.83333%; + } + + .el-col-md-pull-11 { + position: relative; + right: 45.83333%; + } + + .el-col-md-push-11 { + position: relative; + left: 45.83333%; + } + + .el-col-md-12 { + width: 50%; + } + + .el-col-md-offset-12 { + margin-left: 50%; + } + + .el-col-md-pull-12 { + position: relative; + right: 50%; + } + + .el-col-md-push-12 { + position: relative; + left: 50%; + } + + .el-col-md-13 { + width: 54.16667%; + } + + .el-col-md-offset-13 { + margin-left: 54.16667%; + } + + .el-col-md-pull-13 { + position: relative; + right: 54.16667%; + } + + .el-col-md-push-13 { + position: relative; + left: 54.16667%; + } + + .el-col-md-14 { + width: 58.33333%; + } + + .el-col-md-offset-14 { + margin-left: 58.33333%; + } + + .el-col-md-pull-14 { + position: relative; + right: 58.33333%; + } + + .el-col-md-push-14 { + position: relative; + left: 58.33333%; + } + + .el-col-md-15 { + width: 62.5%; + } + + .el-col-md-offset-15 { + margin-left: 62.5%; + } + + .el-col-md-pull-15 { + position: relative; + right: 62.5%; + } + + .el-col-md-push-15 { + position: relative; + left: 62.5%; + } + + .el-col-md-16 { + width: 66.66667%; + } + + .el-col-md-offset-16 { + margin-left: 66.66667%; + } + + .el-col-md-pull-16 { + position: relative; + right: 66.66667%; + } + + .el-col-md-push-16 { + position: relative; + left: 66.66667%; + } + + .el-col-md-17 { + width: 70.83333%; + } + + .el-col-md-offset-17 { + margin-left: 70.83333%; + } + + .el-col-md-pull-17 { + position: relative; + right: 70.83333%; + } + + .el-col-md-push-17 { + position: relative; + left: 70.83333%; + } + + .el-col-md-18 { + width: 75%; + } + + .el-col-md-offset-18 { + margin-left: 75%; + } + + .el-col-md-pull-18 { + position: relative; + right: 75%; + } + + .el-col-md-push-18 { + position: relative; + left: 75%; + } + + .el-col-md-19 { + width: 79.16667%; + } + + .el-col-md-offset-19 { + margin-left: 79.16667%; + } + + .el-col-md-pull-19 { + position: relative; + right: 79.16667%; + } + + .el-col-md-push-19 { + position: relative; + left: 79.16667%; + } + + .el-col-md-20 { + width: 83.33333%; + } + + .el-col-md-offset-20 { + margin-left: 83.33333%; + } + + .el-col-md-pull-20 { + position: relative; + right: 83.33333%; + } + + .el-col-md-push-20 { + position: relative; + left: 83.33333%; + } + + .el-col-md-21 { + width: 87.5%; + } + + .el-col-md-offset-21 { + margin-left: 87.5%; + } + + .el-col-md-pull-21 { + position: relative; + right: 87.5%; + } + + .el-col-md-push-21 { + position: relative; + left: 87.5%; + } + + .el-col-md-22 { + width: 91.66667%; + } + + .el-col-md-offset-22 { + margin-left: 91.66667%; + } + + .el-col-md-pull-22 { + position: relative; + right: 91.66667%; + } + + .el-col-md-push-22 { + position: relative; + left: 91.66667%; + } + + .el-col-md-23 { + width: 95.83333%; + } + + .el-col-md-offset-23 { + margin-left: 95.83333%; + } + + .el-col-md-pull-23 { + position: relative; + right: 95.83333%; + } + + .el-col-md-push-23 { + position: relative; + left: 95.83333%; + } + + .el-col-md-24 { + width: 100%; + } + + .el-col-md-offset-24 { + margin-left: 100%; + } + + .el-col-md-pull-24 { + position: relative; + right: 100%; + } + + .el-col-md-push-24 { + position: relative; + left: 100%; + } +} + +@media only screen and (min-width: 1200px) { + .el-col-lg-0 { + display: none; + width: 0%; + } + + .el-col-lg-offset-0 { + margin-left: 0; + } + + .el-col-lg-pull-0 { + position: relative; + right: 0; + } + + .el-col-lg-push-0 { + position: relative; + left: 0; + } + + .el-col-lg-1 { + width: 4.16667%; + } + + .el-col-lg-offset-1 { + margin-left: 4.16667%; + } + + .el-col-lg-pull-1 { + position: relative; + right: 4.16667%; + } + + .el-col-lg-push-1 { + position: relative; + left: 4.16667%; + } + + .el-col-lg-2 { + width: 8.33333%; + } + + .el-col-lg-offset-2 { + margin-left: 8.33333%; + } + + .el-col-lg-pull-2 { + position: relative; + right: 8.33333%; + } + + .el-col-lg-push-2 { + position: relative; + left: 8.33333%; + } + + .el-col-lg-3 { + width: 12.5%; + } + + .el-col-lg-offset-3 { + margin-left: 12.5%; + } + + .el-col-lg-pull-3 { + position: relative; + right: 12.5%; + } + + .el-col-lg-push-3 { + position: relative; + left: 12.5%; + } + + .el-col-lg-4 { + width: 16.66667%; + } + + .el-col-lg-offset-4 { + margin-left: 16.66667%; + } + + .el-col-lg-pull-4 { + position: relative; + right: 16.66667%; + } + + .el-col-lg-push-4 { + position: relative; + left: 16.66667%; + } + + .el-col-lg-5 { + width: 20.83333%; + } + + .el-col-lg-offset-5 { + margin-left: 20.83333%; + } + + .el-col-lg-pull-5 { + position: relative; + right: 20.83333%; + } + + .el-col-lg-push-5 { + position: relative; + left: 20.83333%; + } + + .el-col-lg-6 { + width: 25%; + } + + .el-col-lg-offset-6 { + margin-left: 25%; + } + + .el-col-lg-pull-6 { + position: relative; + right: 25%; + } + + .el-col-lg-push-6 { + position: relative; + left: 25%; + } + + .el-col-lg-7 { + width: 29.16667%; + } + + .el-col-lg-offset-7 { + margin-left: 29.16667%; + } + + .el-col-lg-pull-7 { + position: relative; + right: 29.16667%; + } + + .el-col-lg-push-7 { + position: relative; + left: 29.16667%; + } + + .el-col-lg-8 { + width: 33.33333%; + } + + .el-col-lg-offset-8 { + margin-left: 33.33333%; + } + + .el-col-lg-pull-8 { + position: relative; + right: 33.33333%; + } + + .el-col-lg-push-8 { + position: relative; + left: 33.33333%; + } + + .el-col-lg-9 { + width: 37.5%; + } + + .el-col-lg-offset-9 { + margin-left: 37.5%; + } + + .el-col-lg-pull-9 { + position: relative; + right: 37.5%; + } + + .el-col-lg-push-9 { + position: relative; + left: 37.5%; + } + + .el-col-lg-10 { + width: 41.66667%; + } + + .el-col-lg-offset-10 { + margin-left: 41.66667%; + } + + .el-col-lg-pull-10 { + position: relative; + right: 41.66667%; + } + + .el-col-lg-push-10 { + position: relative; + left: 41.66667%; + } + + .el-col-lg-11 { + width: 45.83333%; + } + + .el-col-lg-offset-11 { + margin-left: 45.83333%; + } + + .el-col-lg-pull-11 { + position: relative; + right: 45.83333%; + } + + .el-col-lg-push-11 { + position: relative; + left: 45.83333%; + } + + .el-col-lg-12 { + width: 50%; + } + + .el-col-lg-offset-12 { + margin-left: 50%; + } + + .el-col-lg-pull-12 { + position: relative; + right: 50%; + } + + .el-col-lg-push-12 { + position: relative; + left: 50%; + } + + .el-col-lg-13 { + width: 54.16667%; + } + + .el-col-lg-offset-13 { + margin-left: 54.16667%; + } + + .el-col-lg-pull-13 { + position: relative; + right: 54.16667%; + } + + .el-col-lg-push-13 { + position: relative; + left: 54.16667%; + } + + .el-col-lg-14 { + width: 58.33333%; + } + + .el-col-lg-offset-14 { + margin-left: 58.33333%; + } + + .el-col-lg-pull-14 { + position: relative; + right: 58.33333%; + } + + .el-col-lg-push-14 { + position: relative; + left: 58.33333%; + } + + .el-col-lg-15 { + width: 62.5%; + } + + .el-col-lg-offset-15 { + margin-left: 62.5%; + } + + .el-col-lg-pull-15 { + position: relative; + right: 62.5%; + } + + .el-col-lg-push-15 { + position: relative; + left: 62.5%; + } + + .el-col-lg-16 { + width: 66.66667%; + } + + .el-col-lg-offset-16 { + margin-left: 66.66667%; + } + + .el-col-lg-pull-16 { + position: relative; + right: 66.66667%; + } + + .el-col-lg-push-16 { + position: relative; + left: 66.66667%; + } + + .el-col-lg-17 { + width: 70.83333%; + } + + .el-col-lg-offset-17 { + margin-left: 70.83333%; + } + + .el-col-lg-pull-17 { + position: relative; + right: 70.83333%; + } + + .el-col-lg-push-17 { + position: relative; + left: 70.83333%; + } + + .el-col-lg-18 { + width: 75%; + } + + .el-col-lg-offset-18 { + margin-left: 75%; + } + + .el-col-lg-pull-18 { + position: relative; + right: 75%; + } + + .el-col-lg-push-18 { + position: relative; + left: 75%; + } + + .el-col-lg-19 { + width: 79.16667%; + } + + .el-col-lg-offset-19 { + margin-left: 79.16667%; + } + + .el-col-lg-pull-19 { + position: relative; + right: 79.16667%; + } + + .el-col-lg-push-19 { + position: relative; + left: 79.16667%; + } + + .el-col-lg-20 { + width: 83.33333%; + } + + .el-col-lg-offset-20 { + margin-left: 83.33333%; + } + + .el-col-lg-pull-20 { + position: relative; + right: 83.33333%; + } + + .el-col-lg-push-20 { + position: relative; + left: 83.33333%; + } + + .el-col-lg-21 { + width: 87.5%; + } + + .el-col-lg-offset-21 { + margin-left: 87.5%; + } + + .el-col-lg-pull-21 { + position: relative; + right: 87.5%; + } + + .el-col-lg-push-21 { + position: relative; + left: 87.5%; + } + + .el-col-lg-22 { + width: 91.66667%; + } + + .el-col-lg-offset-22 { + margin-left: 91.66667%; + } + + .el-col-lg-pull-22 { + position: relative; + right: 91.66667%; + } + + .el-col-lg-push-22 { + position: relative; + left: 91.66667%; + } + + .el-col-lg-23 { + width: 95.83333%; + } + + .el-col-lg-offset-23 { + margin-left: 95.83333%; + } + + .el-col-lg-pull-23 { + position: relative; + right: 95.83333%; + } + + .el-col-lg-push-23 { + position: relative; + left: 95.83333%; + } + + .el-col-lg-24 { + width: 100%; + } + + .el-col-lg-offset-24 { + margin-left: 100%; + } + + .el-col-lg-pull-24 { + position: relative; + right: 100%; + } + + .el-col-lg-push-24 { + position: relative; + left: 100%; + } +} + +@media only screen and (min-width: 1920px) { + .el-col-xl-0 { + display: none; + width: 0%; + } + + .el-col-xl-offset-0 { + margin-left: 0; + } + + .el-col-xl-pull-0 { + position: relative; + right: 0; + } + + .el-col-xl-push-0 { + position: relative; + left: 0; + } + + .el-col-xl-1 { + width: 4.16667%; + } + + .el-col-xl-offset-1 { + margin-left: 4.16667%; + } + + .el-col-xl-pull-1 { + position: relative; + right: 4.16667%; + } + + .el-col-xl-push-1 { + position: relative; + left: 4.16667%; + } + + .el-col-xl-2 { + width: 8.33333%; + } + + .el-col-xl-offset-2 { + margin-left: 8.33333%; + } + + .el-col-xl-pull-2 { + position: relative; + right: 8.33333%; + } + + .el-col-xl-push-2 { + position: relative; + left: 8.33333%; + } + + .el-col-xl-3 { + width: 12.5%; + } + + .el-col-xl-offset-3 { + margin-left: 12.5%; + } + + .el-col-xl-pull-3 { + position: relative; + right: 12.5%; + } + + .el-col-xl-push-3 { + position: relative; + left: 12.5%; + } + + .el-col-xl-4 { + width: 16.66667%; + } + + .el-col-xl-offset-4 { + margin-left: 16.66667%; + } + + .el-col-xl-pull-4 { + position: relative; + right: 16.66667%; + } + + .el-col-xl-push-4 { + position: relative; + left: 16.66667%; + } + + .el-col-xl-5 { + width: 20.83333%; + } + + .el-col-xl-offset-5 { + margin-left: 20.83333%; + } + + .el-col-xl-pull-5 { + position: relative; + right: 20.83333%; + } + + .el-col-xl-push-5 { + position: relative; + left: 20.83333%; + } + + .el-col-xl-6 { + width: 25%; + } + + .el-col-xl-offset-6 { + margin-left: 25%; + } + + .el-col-xl-pull-6 { + position: relative; + right: 25%; + } + + .el-col-xl-push-6 { + position: relative; + left: 25%; + } + + .el-col-xl-7 { + width: 29.16667%; + } + + .el-col-xl-offset-7 { + margin-left: 29.16667%; + } + + .el-col-xl-pull-7 { + position: relative; + right: 29.16667%; + } + + .el-col-xl-push-7 { + position: relative; + left: 29.16667%; + } + + .el-col-xl-8 { + width: 33.33333%; + } + + .el-col-xl-offset-8 { + margin-left: 33.33333%; + } + + .el-col-xl-pull-8 { + position: relative; + right: 33.33333%; + } + + .el-col-xl-push-8 { + position: relative; + left: 33.33333%; + } + + .el-col-xl-9 { + width: 37.5%; + } + + .el-col-xl-offset-9 { + margin-left: 37.5%; + } + + .el-col-xl-pull-9 { + position: relative; + right: 37.5%; + } + + .el-col-xl-push-9 { + position: relative; + left: 37.5%; + } + + .el-col-xl-10 { + width: 41.66667%; + } + + .el-col-xl-offset-10 { + margin-left: 41.66667%; + } + + .el-col-xl-pull-10 { + position: relative; + right: 41.66667%; + } + + .el-col-xl-push-10 { + position: relative; + left: 41.66667%; + } + + .el-col-xl-11 { + width: 45.83333%; + } + + .el-col-xl-offset-11 { + margin-left: 45.83333%; + } + + .el-col-xl-pull-11 { + position: relative; + right: 45.83333%; + } + + .el-col-xl-push-11 { + position: relative; + left: 45.83333%; + } + + .el-col-xl-12 { + width: 50%; + } + + .el-col-xl-offset-12 { + margin-left: 50%; + } + + .el-col-xl-pull-12 { + position: relative; + right: 50%; + } + + .el-col-xl-push-12 { + position: relative; + left: 50%; + } + + .el-col-xl-13 { + width: 54.16667%; + } + + .el-col-xl-offset-13 { + margin-left: 54.16667%; + } + + .el-col-xl-pull-13 { + position: relative; + right: 54.16667%; + } + + .el-col-xl-push-13 { + position: relative; + left: 54.16667%; + } + + .el-col-xl-14 { + width: 58.33333%; + } + + .el-col-xl-offset-14 { + margin-left: 58.33333%; + } + + .el-col-xl-pull-14 { + position: relative; + right: 58.33333%; + } + + .el-col-xl-push-14 { + position: relative; + left: 58.33333%; + } + + .el-col-xl-15 { + width: 62.5%; + } + + .el-col-xl-offset-15 { + margin-left: 62.5%; + } + + .el-col-xl-pull-15 { + position: relative; + right: 62.5%; + } + + .el-col-xl-push-15 { + position: relative; + left: 62.5%; + } + + .el-col-xl-16 { + width: 66.66667%; + } + + .el-col-xl-offset-16 { + margin-left: 66.66667%; + } + + .el-col-xl-pull-16 { + position: relative; + right: 66.66667%; + } + + .el-col-xl-push-16 { + position: relative; + left: 66.66667%; + } + + .el-col-xl-17 { + width: 70.83333%; + } + + .el-col-xl-offset-17 { + margin-left: 70.83333%; + } + + .el-col-xl-pull-17 { + position: relative; + right: 70.83333%; + } + + .el-col-xl-push-17 { + position: relative; + left: 70.83333%; + } + + .el-col-xl-18 { + width: 75%; + } + + .el-col-xl-offset-18 { + margin-left: 75%; + } + + .el-col-xl-pull-18 { + position: relative; + right: 75%; + } + + .el-col-xl-push-18 { + position: relative; + left: 75%; + } + + .el-col-xl-19 { + width: 79.16667%; + } + + .el-col-xl-offset-19 { + margin-left: 79.16667%; + } + + .el-col-xl-pull-19 { + position: relative; + right: 79.16667%; + } + + .el-col-xl-push-19 { + position: relative; + left: 79.16667%; + } + + .el-col-xl-20 { + width: 83.33333%; + } + + .el-col-xl-offset-20 { + margin-left: 83.33333%; + } + + .el-col-xl-pull-20 { + position: relative; + right: 83.33333%; + } + + .el-col-xl-push-20 { + position: relative; + left: 83.33333%; + } + + .el-col-xl-21 { + width: 87.5%; + } + + .el-col-xl-offset-21 { + margin-left: 87.5%; + } + + .el-col-xl-pull-21 { + position: relative; + right: 87.5%; + } + + .el-col-xl-push-21 { + position: relative; + left: 87.5%; + } + + .el-col-xl-22 { + width: 91.66667%; + } + + .el-col-xl-offset-22 { + margin-left: 91.66667%; + } + + .el-col-xl-pull-22 { + position: relative; + right: 91.66667%; + } + + .el-col-xl-push-22 { + position: relative; + left: 91.66667%; + } + + .el-col-xl-23 { + width: 95.83333%; + } + + .el-col-xl-offset-23 { + margin-left: 95.83333%; + } + + .el-col-xl-pull-23 { + position: relative; + right: 95.83333%; + } + + .el-col-xl-push-23 { + position: relative; + left: 95.83333%; + } + + .el-col-xl-24 { + width: 100%; + } + + .el-col-xl-offset-24 { + margin-left: 100%; + } + + .el-col-xl-pull-24 { + position: relative; + right: 100%; + } + + .el-col-xl-push-24 { + position: relative; + left: 100%; + } +} + +@-webkit-keyframes progress { + 0% { + background-position: 0 0; + } + + 100% { + background-position: 32px 0; + } +} + +.el-upload { + display: inline-block; + text-align: center; + cursor: pointer; + outline: 0; +} + +.el-upload__input { + display: none; +} + +.el-upload__tip { + font-size: 12px; + color: #606266; + margin-top: 7px; +} + +.el-upload iframe { + position: absolute; + z-index: -1; + top: 0; + left: 0; + opacity: 0; + filter: alpha(opacity=0); +} + +.el-upload--picture-card { + background-color: #fbfdff; + border: 1px dashed #c0ccda; + border-radius: 6px; + box-sizing: border-box; + width: 148px; + height: 148px; + line-height: 146px; + vertical-align: top; +} + +.el-upload--picture-card i { + font-size: 28px; + color: #8c939d; +} + +.el-upload--picture-card:hover, +.el-upload:focus { + border-color: #00a5a7; + color: #00a5a7; +} + +.el-upload:focus .el-upload-dragger { + border-color: #00a5a7; +} + +.el-upload-dragger { + background-color: #fff; + border: 1px dashed #d9d9d9; + border-radius: 6px; + box-sizing: border-box; + width: 360px; + height: 180px; + text-align: center; + position: relative; + overflow: hidden; +} + +.el-upload-dragger .el-icon-upload { + font-size: 67px; + color: #c0c4cc; + margin: 40px 0 16px; + line-height: 50px; +} + +.el-upload-dragger + .el-upload__tip { + text-align: center; +} + +.el-upload-dragger ~ .el-upload__files { + border-top: 1px solid #dcdfe6; + margin-top: 7px; + padding-top: 5px; +} + +.el-upload-dragger .el-upload__text { + color: #606266; + font-size: 14px; + text-align: center; +} + +.el-upload-dragger .el-upload__text em { + color: #00a5a7; + font-style: normal; +} + +.el-upload-dragger:hover { + border-color: #00a5a7; +} + +.el-upload-dragger.is-dragover { + background-color: rgba(32, 159, 255, 0.06); + border: 2px dashed #00a5a7; +} + +.el-upload-list { + margin: 0; + padding: 0; + list-style: none; +} + +.el-upload-list__item { + -webkit-transition: all 0.5s cubic-bezier(0.55, 0, 0.1, 1); + transition: all 0.5s cubic-bezier(0.55, 0, 0.1, 1); + font-size: 14px; + color: #606266; + line-height: 1.8; + margin-top: 5px; + position: relative; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-radius: 4px; + width: 100%; +} + +.el-upload-list__item .el-progress { + position: absolute; + top: 20px; + width: 100%; +} + +.el-upload-list__item .el-progress__text { + position: absolute; + right: 0; + top: -13px; +} + +.el-upload-list__item .el-progress-bar { + margin-right: 0; + padding-right: 0; +} + +.el-upload-list__item:first-child { + margin-top: 10px; +} + +.el-upload-list__item .el-icon-upload-success { + color: #67c23a; +} + +.el-upload-list__item .el-icon-close { + display: none; + position: absolute; + top: 5px; + right: 5px; + cursor: pointer; + opacity: 0.75; + color: #606266; +} + +.el-upload-list__item .el-icon-close:hover { + opacity: 1; +} + +.el-upload-list__item .el-icon-close-tip { + display: none; + position: absolute; + top: 5px; + right: 5px; + font-size: 12px; + cursor: pointer; + opacity: 1; + color: #00a5a7; +} + +.el-upload-list__item:hover { + background-color: #f5f7fa; +} + +.el-upload-list__item:hover .el-icon-close { + display: inline-block; +} + +.el-upload-list__item:hover .el-progress__text { + display: none; +} + +.el-upload-list__item.is-success .el-upload-list__item-status-label { + display: block; +} + +.el-upload-list__item.is-success .el-upload-list__item-name:focus, +.el-upload-list__item.is-success .el-upload-list__item-name:hover { + color: #00a5a7; + cursor: pointer; +} + +.el-upload-list__item.is-success:focus:not(:hover) .el-icon-close-tip { + display: inline-block; +} + +.el-upload-list__item.is-success:active .el-icon-close-tip, +.el-upload-list__item.is-success:focus .el-upload-list__item-status-label, +.el-upload-list__item.is-success:hover .el-upload-list__item-status-label, +.el-upload-list__item.is-success:not(.focusing):focus .el-icon-close-tip { + display: none; +} + +.el-upload-list.is-disabled .el-upload-list__item:hover .el-upload-list__item-status-label { + display: block; +} + +.el-upload-list__item-name { + color: #606266; + display: block; + margin-right: 40px; + overflow: hidden; + padding-left: 4px; + text-overflow: ellipsis; + -webkit-transition: color 0.3s; + transition: color 0.3s; + white-space: nowrap; +} + +.el-upload-list__item-name [class^='el-icon'] { + height: 100%; + margin-right: 7px; + color: #909399; + line-height: inherit; +} + +.el-upload-list__item-status-label { + position: absolute; + right: 5px; + top: 0; + line-height: inherit; + display: none; +} + +.el-upload-list__item-delete { + position: absolute; + right: 10px; + top: 0; + font-size: 12px; + color: #606266; + display: none; +} + +.el-upload-list__item-delete:hover { + color: #00a5a7; +} + +.el-upload-list--picture-card { + margin: 0; + display: inline; + vertical-align: top; +} + +.el-upload-list--picture-card .el-upload-list__item { + overflow: hidden; + background-color: #fff; + border: 1px solid #c0ccda; + border-radius: 6px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 148px; + height: 148px; + margin: 0 8px 8px 0; + display: inline-block; +} + +.el-upload-list--picture-card .el-upload-list__item .el-icon-check, +.el-upload-list--picture-card .el-upload-list__item .el-icon-circle-check { + color: #fff; +} + +.el-upload-list--picture-card .el-upload-list__item .el-icon-close, +.el-upload-list--picture-card .el-upload-list__item:hover .el-upload-list__item-status-label { + display: none; +} + +.el-upload-list--picture-card .el-upload-list__item:hover .el-progress__text { + display: block; +} + +.el-upload-list--picture-card .el-upload-list__item-name { + display: none; +} + +.el-upload-list--picture-card .el-upload-list__item-thumbnail { + width: 100%; + height: 100%; +} + +.el-upload-list--picture-card .el-upload-list__item-status-label { + position: absolute; + right: -15px; + top: -6px; + width: 40px; + height: 24px; + background: #13ce66; + text-align: center; + -webkit-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 0 1pc 1px rgba(0, 0, 0, 0.2); + box-shadow: 0 0 1pc 1px rgba(0, 0, 0, 0.2); +} + +.el-upload-list--picture-card .el-upload-list__item-status-label i { + font-size: 12px; + margin-top: 11px; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); +} + +.el-upload-list--picture-card .el-upload-list__item-actions { + position: absolute; + width: 100%; + height: 100%; + left: 0; + top: 0; + cursor: default; + text-align: center; + color: #fff; + opacity: 0; + font-size: 20px; + background-color: rgba(0, 0, 0, 0.5); + -webkit-transition: opacity 0.3s; + transition: opacity 0.3s; +} + +.el-upload-list--picture-card .el-upload-list__item-actions::after { + display: inline-block; + content: ''; + height: 100%; + vertical-align: middle; +} + +.el-upload-list--picture-card .el-upload-list__item-actions span { + display: none; + cursor: pointer; +} + +.el-upload-list--picture-card .el-upload-list__item-actions span + span { + margin-left: 15px; +} + +.el-upload-list--picture-card .el-upload-list__item-actions .el-upload-list__item-delete { + position: static; + font-size: inherit; + color: inherit; +} + +.el-upload-list--picture-card .el-upload-list__item-actions:hover { + opacity: 1; +} + +.el-upload-list--picture-card .el-upload-list__item-actions:hover span { + display: inline-block; +} + +.el-upload-list--picture-card .el-progress { + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + bottom: auto; + width: 126px; +} + +.el-upload-list--picture-card .el-progress .el-progress__text { + top: 50%; +} + +.el-upload-list--picture .el-upload-list__item { + overflow: hidden; + z-index: 0; + background-color: #fff; + border: 1px solid #c0ccda; + border-radius: 6px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + margin-top: 10px; + padding: 10px 10px 10px 90px; + height: 92px; +} + +.el-upload-list--picture .el-upload-list__item .el-icon-check, +.el-upload-list--picture .el-upload-list__item .el-icon-circle-check { + color: #fff; +} + +.el-upload-list--picture .el-upload-list__item:hover .el-upload-list__item-status-label { + background: 0 0; + -webkit-box-shadow: none; + box-shadow: none; + top: -2px; + right: -12px; +} + +.el-upload-list--picture .el-upload-list__item:hover .el-progress__text { + display: block; +} + +.el-upload-list--picture .el-upload-list__item.is-success .el-upload-list__item-name { + line-height: 70px; + margin-top: 0; +} + +.el-upload-list--picture .el-upload-list__item.is-success .el-upload-list__item-name i { + display: none; +} + +.el-upload-list--picture .el-upload-list__item-thumbnail { + vertical-align: middle; + display: inline-block; + width: 70px; + height: 70px; + float: left; + position: relative; + z-index: 1; + margin-left: -80px; + background-color: #fff; +} + +.el-upload-list--picture .el-upload-list__item-name { + display: block; + margin-top: 20px; +} + +.el-upload-list--picture .el-upload-list__item-name i { + font-size: 70px; + line-height: 1; + position: absolute; + left: 9px; + top: 10px; +} + +.el-upload-list--picture .el-upload-list__item-status-label { + position: absolute; + right: -17px; + top: -7px; + width: 46px; + height: 26px; + background: #13ce66; + text-align: center; + -webkit-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 1px 1px #ccc; + box-shadow: 0 1px 1px #ccc; +} + +.el-upload-list--picture .el-upload-list__item-status-label i { + font-size: 12px; + margin-top: 12px; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); +} + +.el-upload-list--picture .el-progress { + position: relative; + top: -7px; +} + +.el-upload-cover { + position: absolute; + left: 0; + top: 0; + width: 100%; + height: 100%; + overflow: hidden; + z-index: 10; + cursor: default; +} + +.el-upload-cover::after { + display: inline-block; + height: 100%; + vertical-align: middle; +} + +.el-upload-cover img { + display: block; + width: 100%; + height: 100%; +} + +.el-upload-cover__label { + position: absolute; + right: -15px; + top: -6px; + width: 40px; + height: 24px; + background: #13ce66; + text-align: center; + -webkit-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 0 1pc 1px rgba(0, 0, 0, 0.2); + box-shadow: 0 0 1pc 1px rgba(0, 0, 0, 0.2); +} + +.el-upload-cover__label i { + font-size: 12px; + margin-top: 11px; + -webkit-transform: rotate(-45deg); + transform: rotate(-45deg); + color: #fff; +} + +.el-upload-cover__progress { + display: inline-block; + vertical-align: middle; + position: static; + width: 243px; +} + +.el-upload-cover__progress + .el-upload__inner { + opacity: 0; +} + +.el-upload-cover__content { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; +} + +.el-upload-cover__interact { + position: absolute; + bottom: 0; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(0, 0, 0, 0.72); + text-align: center; +} + +.el-upload-cover__interact .btn { + display: inline-block; + color: #fff; + font-size: 14px; + cursor: pointer; + vertical-align: middle; + -webkit-transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + margin-top: 60px; +} + +.el-upload-cover__interact .btn span { + opacity: 0; + -webkit-transition: opacity 0.15s linear; + transition: opacity 0.15s linear; +} + +.el-upload-cover__interact .btn:not(:first-child) { + margin-left: 35px; +} + +.el-upload-cover__interact .btn:hover { + -webkit-transform: translateY(-13px); + transform: translateY(-13px); +} + +.el-upload-cover__interact .btn:hover span { + opacity: 1; +} + +.el-upload-cover__interact .btn i { + color: #fff; + display: block; + font-size: 24px; + line-height: inherit; + margin: 0 auto 5px; +} + +.el-upload-cover__title { + position: absolute; + bottom: 0; + left: 0; + background-color: #fff; + height: 36px; + width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-weight: 400; + text-align: left; + padding: 0 10px; + margin: 0; + line-height: 36px; + font-size: 14px; + color: #303133; +} + +.el-upload-cover + .el-upload__inner { + opacity: 0; + position: relative; + z-index: 1; +} + +.el-progress { + position: relative; + line-height: 1; +} + +.el-progress__text { + font-size: 14px; + color: #606266; + display: inline-block; + vertical-align: middle; + margin-left: 10px; + line-height: 1; +} + +.el-progress__text i { + vertical-align: middle; + display: block; +} + +.el-progress--circle, +.el-progress--dashboard { + display: inline-block; +} + +.el-progress--circle .el-progress__text, +.el-progress--dashboard .el-progress__text { + position: absolute; + top: 50%; + left: 0; + width: 100%; + text-align: center; + margin: 0; + -webkit-transform: translate(0, -50%); + transform: translate(0, -50%); +} + +.el-progress--circle .el-progress__text i, +.el-progress--dashboard .el-progress__text i { + vertical-align: middle; + display: inline-block; +} + +.el-progress--without-text .el-progress__text { + display: none; +} + +.el-progress--without-text .el-progress-bar { + padding-right: 0; + margin-right: 0; + display: block; +} + +.el-progress-bar, +.el-progress-bar__inner::after, +.el-progress-bar__innerText, +.el-spinner { + display: inline-block; + vertical-align: middle; +} + +.el-progress--text-inside .el-progress-bar { + padding-right: 0; + margin-right: 0; +} + +.el-progress.is-success .el-progress-bar__inner { + background-color: #67c23a; +} + +.el-progress.is-success .el-progress__text { + color: #67c23a; +} + +.el-progress.is-warning .el-progress-bar__inner { + background-color: #e6a23c; +} + +.el-progress.is-warning .el-progress__text { + color: #e6a23c; +} + +.el-progress.is-exception .el-progress-bar__inner { + background-color: #f56c6c; +} + +.el-progress.is-exception .el-progress__text { + color: #f56c6c; +} + +.el-progress-bar { + padding-right: 50px; + width: 100%; + margin-right: -55px; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-progress-bar__outer { + height: 6px; + border-radius: 100px; + background-color: #ebeef5; + overflow: hidden; + position: relative; + vertical-align: middle; +} + +.el-progress-bar__inner { + position: absolute; + left: 0; + top: 0; + height: 100%; + background-color: #00a5a7; + text-align: right; + border-radius: 100px; + line-height: 1; + white-space: nowrap; + -webkit-transition: width 0.6s ease; + transition: width 0.6s ease; +} + +.el-card, +.el-message { + border-radius: 4px; + overflow: hidden; +} + +.el-progress-bar__inner::after { + height: 100%; +} + +.el-progress-bar__innerText { + color: #fff; + font-size: 12px; + margin: 0 5px; +} + +@keyframes progress { + 0% { + background-position: 0 0; + } + + 100% { + background-position: 32px 0; + } +} + +.el-time-spinner { + width: 100%; + white-space: nowrap; +} + +.el-spinner-inner { + -webkit-animation: rotate 2s linear infinite; + animation: rotate 2s linear infinite; + width: 50px; + height: 50px; +} + +.el-spinner-inner .path { + stroke: #ececec; + stroke-linecap: round; + -webkit-animation: dash 1.5s ease-in-out infinite; + animation: dash 1.5s ease-in-out infinite; +} + +@-webkit-keyframes rotate { + 100% { + -webkit-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +@keyframes rotate { + 100% { + -webkit-transform: rotate(360deg); + transform: rotate(360deg); + } +} + +@-webkit-keyframes dash { + 0% { + stroke-dasharray: 1, 150; + stroke-dashoffset: 0; + } + + 50% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -35; + } + + 100% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -124; + } +} + +@keyframes dash { + 0% { + stroke-dasharray: 1, 150; + stroke-dashoffset: 0; + } + + 50% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -35; + } + + 100% { + stroke-dasharray: 90, 150; + stroke-dashoffset: -124; + } +} + +.el-message { + min-width: 380px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-width: 1px; + border-style: solid; + border-color: #ebeef5; + position: fixed; + left: 50%; + top: 20px; + -webkit-transform: translateX(-50%); + transform: translateX(-50%); + background-color: #edf2fc; + -webkit-transition: opacity 0.3s, top 0.4s, -webkit-transform 0.4s; + transition: opacity 0.3s, top 0.4s, -webkit-transform 0.4s; + transition: opacity 0.3s, transform 0.4s, top 0.4s; + transition: opacity 0.3s, transform 0.4s, top 0.4s, -webkit-transform 0.4s; + padding: 15px 15px 15px 20px; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-message.is-center { + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; +} + +.el-message.is-closable .el-message__content { + padding-right: 16px; +} + +.el-message p { + margin: 0; +} + +.el-message--info .el-message__content { + color: #909399; +} + +.el-message--success { + background-color: #f0f9eb; + border-color: #e1f3d8; +} + +.el-message--success .el-message__content { + color: #67c23a; +} + +.el-message--warning { + background-color: #fdf6ec; + border-color: #faecd8; +} + +.el-message--warning .el-message__content { + color: #e6a23c; +} + +.el-message--error { + background-color: #fef0f0; + border-color: #fde2e2; +} + +.el-message--error .el-message__content { + color: #f56c6c; +} + +.el-message__icon { + margin-right: 10px; +} + +.el-message__content { + padding: 0; + font-size: 14px; + line-height: 1; +} + +.el-message__closeBtn { + position: absolute; + top: 50%; + right: 15px; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + cursor: pointer; + color: #c0c4cc; + font-size: 16px; +} + +.el-message__closeBtn:hover { + color: #909399; +} + +.el-message .el-icon-success { + color: #67c23a; +} + +.el-message .el-icon-error { + color: #f56c6c; +} + +.el-message .el-icon-info { + color: #909399; +} + +.el-message .el-icon-warning { + color: #e6a23c; +} + +.el-message-fade-enter, +.el-message-fade-leave-active { + opacity: 0; + -webkit-transform: translate(-50%, -100%); + transform: translate(-50%, -100%); +} + +.el-badge { + position: relative; + vertical-align: middle; + display: inline-block; +} + +.el-badge__content { + background-color: #f56c6c; + border-radius: 10px; + color: #fff; + display: inline-block; + font-size: 12px; + height: 18px; + line-height: 18px; + padding: 0 6px; + text-align: center; + white-space: nowrap; + border: 1px solid #fff; +} + +.el-badge__content.is-fixed { + position: absolute; + top: 0; + right: 10px; + -webkit-transform: translateY(-50%) translateX(100%); + transform: translateY(-50%) translateX(100%); +} + +.el-rate__icon, +.el-rate__item { + position: relative; + display: inline-block; +} + +.el-badge__content.is-fixed.is-dot { + right: 5px; +} + +.el-badge__content.is-dot { + height: 8px; + width: 8px; + padding: 0; + right: 0; + border-radius: 50%; +} + +.el-badge__content--primary { + background-color: #00a5a7; +} + +.el-badge__content--success { + background-color: #67c23a; +} + +.el-badge__content--warning { + background-color: #e6a23c; +} + +.el-badge__content--info { + background-color: #909399; +} + +.el-badge__content--danger { + background-color: #f56c6c; +} + +.el-card { + border: 1px solid #ebeef5; + background-color: #fff; + color: #303133; + -webkit-transition: 0.3s; + transition: 0.3s; +} + +.el-card.is-always-shadow, +.el-card.is-hover-shadow:focus, +.el-card.is-hover-shadow:hover { + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-card__header { + padding: 18px 20px; + border-bottom: 1px solid #ebeef5; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-card__body { + padding: 20px; +} + +.el-rate { + height: 20px; + line-height: 1; +} + +.el-rate__item { + font-size: 0; + vertical-align: middle; +} + +.el-rate__icon { + font-size: 18px; + margin-right: 6px; + color: #c0c4cc; + -webkit-transition: 0.3s; + transition: 0.3s; +} + +.el-rate__decimal, +.el-rate__icon .path2 { + position: absolute; + top: 0; + left: 0; +} + +.el-rate__icon.hover { + -webkit-transform: scale(1.15); + transform: scale(1.15); +} + +.el-rate__decimal { + display: inline-block; + overflow: hidden; +} + +.el-step.is-vertical, +.el-steps { + display: -webkit-box; + display: -ms-flexbox; +} + +.el-rate__text { + font-size: 14px; + vertical-align: middle; +} + +.el-steps { + display: flex; +} + +.el-steps--simple { + padding: 13px 8%; + border-radius: 4px; + background: #f5f7fa; +} + +.el-steps--horizontal { + white-space: nowrap; +} + +.el-steps--vertical { + height: 100%; + -webkit-box-orient: vertical; + -ms-flex-flow: column; + flex-flow: column; +} + +.el-step { + position: relative; + -ms-flex-negative: 1; + flex-shrink: 1; +} + +.el-step:last-of-type .el-step__line { + display: none; +} + +.el-step:last-of-type.is-flex { + -ms-flex-preferred-size: auto !important; + flex-basis: auto !important; + -ms-flex-negative: 0; + flex-shrink: 0; + -webkit-box-flex: 0; + -ms-flex-positive: 0; + flex-grow: 0; +} + +.el-step:last-of-type .el-step__description, +.el-step:last-of-type .el-step__main { + padding-right: 0; +} + +.el-step__head { + position: relative; + width: 100%; +} + +.el-step__head.is-process { + color: #303133; + border-color: #303133; +} + +.el-step__head.is-wait { + color: #c0c4cc; + border-color: #c0c4cc; +} + +.el-step__head.is-success { + color: #67c23a; + border-color: #67c23a; +} + +.el-step__head.is-error { + color: #f56c6c; + border-color: #f56c6c; +} + +.el-step__head.is-finish { + color: #00a5a7; + border-color: #00a5a7; +} + +.el-step__icon { + position: relative; + z-index: 1; + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + width: 24px; + height: 24px; + font-size: 14px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + background: #fff; + -webkit-transition: 0.15s ease-out; + transition: 0.15s ease-out; +} + +.el-step__icon.is-text { + border-radius: 50%; + border: 2px solid; + border-color: inherit; +} + +.el-step__icon.is-icon { + width: 40px; +} + +.el-step__icon-inner { + display: inline-block; + user-select: none; + text-align: center; + font-weight: 700; + line-height: 1; + color: inherit; +} + +.el-button, +.el-checkbox, +.el-image-viewer__btn { + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; +} + +.el-step__icon-inner[class*='el-icon']:not(.is-status) { + font-size: 25px; + font-weight: 400; +} + +.el-step__icon-inner.is-status { + -webkit-transform: translateY(1px); + transform: translateY(1px); +} + +.el-step__line { + position: absolute; + border-color: inherit; + background-color: #c0c4cc; +} + +.el-step__line-inner { + display: block; + border-width: 1px; + border-style: solid; + border-color: inherit; + -webkit-transition: 0.15s ease-out; + transition: 0.15s ease-out; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 0; + height: 0; +} + +.el-step__main { + white-space: normal; + text-align: left; +} + +.el-step__title { + font-size: 16px; + line-height: 38px; +} + +.el-step__title.is-process { + font-weight: 700; + color: #303133; +} + +.el-step__title.is-wait { + color: #c0c4cc; +} + +.el-step__title.is-success { + color: #67c23a; +} + +.el-step__title.is-error { + color: #f56c6c; +} + +.el-step__title.is-finish { + color: #00a5a7; +} + +.el-step__description { + padding-right: 10%; + margin-top: -5px; + font-size: 12px; + line-height: 20px; + font-weight: 400; +} + +.el-step__description.is-process { + color: #303133; +} + +.el-step__description.is-wait { + color: #c0c4cc; +} + +.el-step__description.is-success { + color: #67c23a; +} + +.el-step__description.is-error { + color: #f56c6c; +} + +.el-step__description.is-finish { + color: #00a5a7; +} + +.el-step.is-horizontal { + display: inline-block; +} + +.el-step.is-horizontal .el-step__line { + height: 2px; + top: 11px; + left: 0; + right: 0; +} + +.el-step.is-vertical { + display: flex; +} + +.el-step.is-vertical .el-step__head { + -webkit-box-flex: 0; + -ms-flex-positive: 0; + flex-grow: 0; + width: 24px; +} + +.el-step.is-vertical .el-step__main { + padding-left: 10px; + -webkit-box-flex: 1; + -ms-flex-positive: 1; + flex-grow: 1; +} + +.el-step.is-vertical .el-step__title { + line-height: 24px; + padding-bottom: 8px; +} + +.el-step.is-vertical .el-step__line { + width: 2px; + top: 0; + bottom: 0; + left: 11px; +} + +.el-step.is-vertical .el-step__icon.is-icon { + width: 24px; +} + +.el-step.is-center .el-step__head, +.el-step.is-center .el-step__main { + text-align: center; +} + +.el-step.is-center .el-step__description { + padding-left: 20%; + padding-right: 20%; +} + +.el-step.is-center .el-step__line { + left: 50%; + right: -50%; +} + +.el-step.is-simple { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-step.is-simple .el-step__head { + width: auto; + font-size: 0; + padding-right: 10px; +} + +.el-step.is-simple .el-step__icon { + background: 0 0; + width: 16px; + height: 16px; + font-size: 12px; +} + +.el-step.is-simple .el-step__icon-inner[class*='el-icon']:not(.is-status) { + font-size: 18px; +} + +.el-step.is-simple .el-step__icon-inner.is-status { + -webkit-transform: scale(0.8) translateY(1px); + transform: scale(0.8) translateY(1px); +} + +.el-step.is-simple .el-step__main { + position: relative; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: stretch; + -ms-flex-align: stretch; + align-items: stretch; + -webkit-box-flex: 1; + -ms-flex-positive: 1; + flex-grow: 1; +} + +.el-step.is-simple .el-step__title { + font-size: 16px; + line-height: 20px; +} + +.el-step.is-simple:not(:last-of-type) .el-step__title { + max-width: 50%; + word-break: break-all; +} + +.el-step.is-simple .el-step__arrow { + -webkit-box-flex: 1; + -ms-flex-positive: 1; + flex-grow: 1; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; +} + +.el-step.is-simple .el-step__arrow::after, +.el-step.is-simple .el-step__arrow::before { + content: ''; + display: inline-block; + position: absolute; + height: 15px; + width: 1px; + background: #c0c4cc; +} + +.el-step.is-simple .el-step__arrow::before { + -webkit-transform: rotate(-45deg) translateY(-4px); + transform: rotate(-45deg) translateY(-4px); + -webkit-transform-origin: 0 0; + transform-origin: 0 0; +} + +.el-step.is-simple .el-step__arrow::after { + -webkit-transform: rotate(45deg) translateY(4px); + transform: rotate(45deg) translateY(4px); + -webkit-transform-origin: 100% 100%; + transform-origin: 100% 100%; +} + +.el-step.is-simple:last-of-type .el-step__arrow { + display: none; +} + +.el-carousel { + position: relative; +} + +.el-carousel--horizontal { + overflow-x: hidden; +} + +.el-carousel--vertical { + overflow-y: hidden; +} + +.el-carousel__container { + position: relative; + height: 300px; +} + +.el-carousel__arrow { + border: none; + outline: 0; + padding: 0; + margin: 0; + height: 36px; + width: 36px; + cursor: pointer; + -webkit-transition: 0.3s; + transition: 0.3s; + border-radius: 50%; + background-color: rgba(31, 45, 61, 0.11); + color: #fff; + position: absolute; + top: 50%; + z-index: 10; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + text-align: center; + font-size: 12px; +} + +.el-carousel__arrow--left { + left: 16px; +} + +.el-carousel__arrow--right { + right: 16px; +} + +.el-carousel__arrow:hover { + background-color: rgba(31, 45, 61, 0.23); +} + +.el-carousel__arrow i { + cursor: pointer; +} + +.el-carousel__indicators { + position: absolute; + list-style: none; + margin: 0; + padding: 0; + z-index: 2; +} + +.el-carousel__indicators--horizontal { + bottom: 0; + left: 50%; + -webkit-transform: translateX(-50%); + transform: translateX(-50%); +} + +.el-carousel__indicators--vertical { + right: 0; + top: 50%; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); +} + +.el-carousel__indicators--outside { + bottom: 26px; + text-align: center; + position: static; + -webkit-transform: none; + transform: none; +} + +.el-carousel__indicators--outside .el-carousel__indicator:hover button { + opacity: 0.64; +} + +.el-carousel__indicators--outside button { + background-color: #c0c4cc; + opacity: 0.24; +} + +.el-carousel__indicators--labels { + left: 0; + right: 0; + -webkit-transform: none; + transform: none; + text-align: center; +} + +.el-carousel__indicators--labels .el-carousel__button { + height: auto; + width: auto; + padding: 2px 18px; + font-size: 12px; +} + +.el-carousel__indicators--labels .el-carousel__indicator { + padding: 6px 4px; +} + +.el-carousel__indicator { + background-color: transparent; + cursor: pointer; +} + +.el-carousel__indicator:hover button { + opacity: 0.72; +} + +.el-carousel__indicator--horizontal { + display: inline-block; + padding: 12px 4px; +} + +.el-carousel__indicator--vertical { + padding: 4px 12px; +} + +.el-carousel__indicator--vertical .el-carousel__button { + width: 2px; + height: 15px; +} + +.el-carousel__indicator.is-active button { + opacity: 1; +} + +.el-carousel__button { + display: block; + opacity: 0.48; + width: 30px; + height: 2px; + background-color: #fff; + border: none; + outline: 0; + padding: 0; + margin: 0; + cursor: pointer; + -webkit-transition: 0.3s; + transition: 0.3s; +} + +.el-carousel__item, +.el-carousel__mask { + height: 100%; + top: 0; + left: 0; + position: absolute; +} + +.carousel-arrow-left-enter, +.carousel-arrow-left-leave-active { + -webkit-transform: translateY(-50%) translateX(-10px); + transform: translateY(-50%) translateX(-10px); + opacity: 0; +} + +.carousel-arrow-right-enter, +.carousel-arrow-right-leave-active { + -webkit-transform: translateY(-50%) translateX(10px); + transform: translateY(-50%) translateX(10px); + opacity: 0; +} + +.el-carousel__item { + width: 100%; + display: inline-block; + overflow: hidden; + z-index: 0; +} + +.el-carousel__item.is-active { + z-index: 2; +} + +.el-carousel__item.is-animating { + -webkit-transition: -webkit-transform 0.4s ease-in-out; + transition: -webkit-transform 0.4s ease-in-out; + transition: transform 0.4s ease-in-out; + transition: transform 0.4s ease-in-out, -webkit-transform 0.4s ease-in-out; +} + +.el-carousel__item--card { + width: 50%; + -webkit-transition: -webkit-transform 0.4s ease-in-out; + transition: -webkit-transform 0.4s ease-in-out; + transition: transform 0.4s ease-in-out; + transition: transform 0.4s ease-in-out, -webkit-transform 0.4s ease-in-out; +} + +.el-carousel__item--card.is-in-stage { + cursor: pointer; + z-index: 1; +} + +.el-carousel__item--card.is-in-stage.is-hover .el-carousel__mask, +.el-carousel__item--card.is-in-stage:hover .el-carousel__mask { + opacity: 0.12; +} + +.el-carousel__item--card.is-active { + z-index: 2; +} + +.el-carousel__mask { + width: 100%; + background-color: #fff; + opacity: 0.24; + -webkit-transition: 0.2s; + transition: 0.2s; +} + +.el-fade-in-enter, +.el-fade-in-leave-active, +.el-fade-in-linear-enter, +.el-fade-in-linear-leave, +.el-fade-in-linear-leave-active, +.fade-in-linear-enter, +.fade-in-linear-leave, +.fade-in-linear-leave-active { + opacity: 0; +} + +.fade-in-linear-enter-active, +.fade-in-linear-leave-active { + -webkit-transition: opacity 0.2s linear; + transition: opacity 0.2s linear; +} + +.el-fade-in-linear-enter-active, +.el-fade-in-linear-leave-active { + -webkit-transition: opacity 0.2s linear; + transition: opacity 0.2s linear; +} + +.el-fade-in-enter-active, +.el-fade-in-leave-active { + -webkit-transition: all 0.3s cubic-bezier(0.55, 0, 0.1, 1); + transition: all 0.3s cubic-bezier(0.55, 0, 0.1, 1); +} + +.el-zoom-in-center-enter-active, +.el-zoom-in-center-leave-active { + -webkit-transition: all 0.3s cubic-bezier(0.55, 0, 0.1, 1); + transition: all 0.3s cubic-bezier(0.55, 0, 0.1, 1); +} + +.el-zoom-in-center-enter, +.el-zoom-in-center-leave-active { + opacity: 0; + -webkit-transform: scaleX(0); + transform: scaleX(0); +} + +.el-zoom-in-top-enter-active, +.el-zoom-in-top-leave-active { + opacity: 1; + -webkit-transform: scaleY(1); + transform: scaleY(1); + -webkit-transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + -webkit-transform-origin: center top; + transform-origin: center top; +} + +.el-zoom-in-top-enter, +.el-zoom-in-top-leave-active { + opacity: 0; + -webkit-transform: scaleY(0); + transform: scaleY(0); +} + +.el-zoom-in-bottom-enter-active, +.el-zoom-in-bottom-leave-active { + opacity: 1; + -webkit-transform: scaleY(1); + transform: scaleY(1); + -webkit-transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + -webkit-transform-origin: center bottom; + transform-origin: center bottom; +} + +.el-zoom-in-bottom-enter, +.el-zoom-in-bottom-leave-active { + opacity: 0; + -webkit-transform: scaleY(0); + transform: scaleY(0); +} + +.el-zoom-in-left-enter-active, +.el-zoom-in-left-leave-active { + opacity: 1; + -webkit-transform: scale(1, 1); + transform: scale(1, 1); + -webkit-transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1); + transition: transform 0.3s cubic-bezier(0.23, 1, 0.32, 1), + opacity 0.3s cubic-bezier(0.23, 1, 0.32, 1), + -webkit-transform 0.3s cubic-bezier(0.23, 1, 0.32, 1); + -webkit-transform-origin: top left; + transform-origin: top left; +} + +.el-zoom-in-left-enter, +.el-zoom-in-left-leave-active { + opacity: 0; + -webkit-transform: scale(0.45, 0.45); + transform: scale(0.45, 0.45); +} + +.collapse-transition { + -webkit-transition: 0.3s height ease-in-out, 0.3s padding-top ease-in-out, + 0.3s padding-bottom ease-in-out; + transition: 0.3s height ease-in-out, 0.3s padding-top ease-in-out, + 0.3s padding-bottom ease-in-out; +} + +.horizontal-collapse-transition { + -webkit-transition: 0.3s width ease-in-out, 0.3s padding-left ease-in-out, + 0.3s padding-right ease-in-out; + transition: 0.3s width ease-in-out, 0.3s padding-left ease-in-out, + 0.3s padding-right ease-in-out; +} + +.el-list-enter-active, +.el-list-leave-active { + -webkit-transition: all 1s; + transition: all 1s; +} + +.el-list-enter, +.el-list-leave-active { + opacity: 0; + -webkit-transform: translateY(-30px); + transform: translateY(-30px); +} + +.el-opacity-transition { + -webkit-transition: opacity 0.3s cubic-bezier(0.55, 0, 0.1, 1); + transition: opacity 0.3s cubic-bezier(0.55, 0, 0.1, 1); +} + +.el-collapse { + border-top: 1px solid #ebeef5; + border-bottom: 1px solid #ebeef5; +} + +.el-collapse-item.is-disabled .el-collapse-item__header { + color: #bbb; + cursor: not-allowed; +} + +.el-collapse-item__header { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + height: 48px; + line-height: 48px; + background-color: #fff; + color: #303133; + cursor: pointer; + border-bottom: 1px solid #ebeef5; + font-size: 13px; + font-weight: 500; + -webkit-transition: border-bottom-color 0.3s; + transition: border-bottom-color 0.3s; + outline: 0; +} + +.el-collapse-item__arrow { + margin: 0 8px 0 auto; + transition: -webkit-transform 0.3s; + transition: transform 0.3s; + transition: transform 0.3s, -webkit-transform 0.3s; + font-weight: 300; +} + +.el-collapse-item__arrow.is-active { + -webkit-transform: rotate(90deg); + transform: rotate(90deg); +} + +.el-collapse-item__header.focusing:focus:not(:hover) { + color: #00a5a7; +} + +.el-collapse-item__header.is-active { + border-bottom-color: transparent; +} + +.el-collapse-item__wrap { + will-change: height; + background-color: #fff; + overflow: hidden; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-bottom: 1px solid #ebeef5; +} + +.el-cascader__tags, +.el-tag { + -webkit-box-sizing: border-box; +} + +.el-collapse-item__content { + padding-bottom: 25px; + font-size: 13px; + color: #303133; + line-height: 1.769230769230769; +} + +.el-collapse-item:last-child { + margin-bottom: -1px; +} + +.el-popper .popper__arrow, +.el-popper .popper__arrow::after { + position: absolute; + display: block; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; +} + +.el-popper .popper__arrow { + border-width: 6px; + -webkit-filter: drop-shadow(0 2px 12px rgba(0, 0, 0, 0.03)); + filter: drop-shadow(0 2px 12px rgba(0, 0, 0, 0.03)); +} + +.el-popper .popper__arrow::after { + content: ' '; + border-width: 6px; +} + +.el-popper[x-placement^='top'] { + margin-bottom: 12px; +} + +.el-popper[x-placement^='top'] .popper__arrow { + bottom: -6px; + left: 50%; + margin-right: 3px; + border-top-color: #ebeef5; + border-bottom-width: 0; +} + +.el-popper[x-placement^='top'] .popper__arrow::after { + bottom: 1px; + margin-left: -6px; + border-top-color: #fff; + border-bottom-width: 0; +} + +.el-popper[x-placement^='bottom'] { + margin-top: 12px; +} + +.el-popper[x-placement^='bottom'] .popper__arrow { + top: -6px; + left: 50%; + margin-right: 3px; + border-top-width: 0; + border-bottom-color: #ebeef5; +} + +.el-popper[x-placement^='bottom'] .popper__arrow::after { + top: 1px; + margin-left: -6px; + border-top-width: 0; + border-bottom-color: #fff; +} + +.el-popper[x-placement^='right'] { + margin-left: 12px; +} + +.el-popper[x-placement^='right'] .popper__arrow { + top: 50%; + left: -6px; + margin-bottom: 3px; + border-right-color: #ebeef5; + border-left-width: 0; +} + +.el-popper[x-placement^='right'] .popper__arrow::after { + bottom: -6px; + left: 1px; + border-right-color: #fff; + border-left-width: 0; +} + +.el-popper[x-placement^='left'] { + margin-right: 12px; +} + +.el-popper[x-placement^='left'] .popper__arrow { + top: 50%; + right: -6px; + margin-bottom: 3px; + border-right-width: 0; + border-left-color: #ebeef5; +} + +.el-popper[x-placement^='left'] .popper__arrow::after { + right: 1px; + bottom: -6px; + margin-left: -6px; + border-right-width: 0; + border-left-color: #fff; +} + +.el-tag { + background-color: rgb(230, 246, 246); + border-color: rgb(204, 237, 237); + display: inline-block; + height: 32px; + padding: 0 10px; + line-height: 30px; + font-size: 12px; + color: #00a5a7; + border-width: 1px; + border-style: solid; + border-radius: 4px; + box-sizing: border-box; + white-space: nowrap; +} + +.el-tag.is-hit { + border-color: #00a5a7; +} + +.el-tag .el-tag__close { + color: #00a5a7; +} + +.el-tag .el-tag__close:hover { + color: #fff; + background-color: #00a5a7; +} + +.el-tag.el-tag--info { + background-color: #f4f4f5; + border-color: #e9e9eb; + color: #909399; +} + +.el-tag.el-tag--info.is-hit { + border-color: #909399; +} + +.el-tag.el-tag--info .el-tag__close { + color: #909399; +} + +.el-tag.el-tag--info .el-tag__close:hover { + color: #fff; + background-color: #909399; +} + +.el-tag.el-tag--success { + background-color: #f0f9eb; + border-color: #e1f3d8; + color: #67c23a; +} + +.el-tag.el-tag--success.is-hit { + border-color: #67c23a; +} + +.el-tag.el-tag--success .el-tag__close { + color: #67c23a; +} + +.el-tag.el-tag--success .el-tag__close:hover { + color: #fff; + background-color: #67c23a; +} + +.el-tag.el-tag--warning { + background-color: #fdf6ec; + border-color: #faecd8; + color: #e6a23c; +} + +.el-tag.el-tag--warning.is-hit { + border-color: #e6a23c; +} + +.el-tag.el-tag--warning .el-tag__close { + color: #e6a23c; +} + +.el-tag.el-tag--warning .el-tag__close:hover { + color: #fff; + background-color: #e6a23c; +} + +.el-tag.el-tag--danger { + background-color: #fef0f0; + border-color: #fde2e2; + color: #f56c6c; +} + +.el-tag.el-tag--danger.is-hit { + border-color: #f56c6c; +} + +.el-tag.el-tag--danger .el-tag__close { + color: #f56c6c; +} + +.el-tag.el-tag--danger .el-tag__close:hover { + color: #fff; + background-color: #f56c6c; +} + +.el-tag .el-icon-close { + border-radius: 50%; + text-align: center; + position: relative; + cursor: pointer; + font-size: 12px; + height: 16px; + width: 16px; + line-height: 16px; + vertical-align: middle; + top: -1px; + right: -5px; +} + +.el-tag .el-icon-close::before { + display: block; +} + +.el-tag--dark { + background-color: #00a5a7; + border-color: #00a5a7; + color: #fff; +} + +.el-tag--dark.is-hit { + border-color: #00a5a7; +} + +.el-tag--dark .el-tag__close { + color: #fff; +} + +.el-tag--dark .el-tag__close:hover { + color: #fff; + background-color: rgb(51, 183, 185); +} + +.el-tag--dark.el-tag--info { + background-color: #909399; + border-color: #909399; + color: #fff; +} + +.el-tag--dark.el-tag--info.is-hit { + border-color: #909399; +} + +.el-tag--dark.el-tag--info .el-tag__close { + color: #fff; +} + +.el-tag--dark.el-tag--info .el-tag__close:hover { + color: #fff; + background-color: #a6a9ad; +} + +.el-tag--dark.el-tag--success { + background-color: #67c23a; + border-color: #67c23a; + color: #fff; +} + +.el-tag--dark.el-tag--success.is-hit { + border-color: #67c23a; +} + +.el-tag--dark.el-tag--success .el-tag__close { + color: #fff; +} + +.el-tag--dark.el-tag--success .el-tag__close:hover { + color: #fff; + background-color: #85ce61; +} + +.el-tag--dark.el-tag--warning { + background-color: #e6a23c; + border-color: #e6a23c; + color: #fff; +} + +.el-tag--dark.el-tag--warning.is-hit { + border-color: #e6a23c; +} + +.el-tag--dark.el-tag--warning .el-tag__close { + color: #fff; +} + +.el-tag--dark.el-tag--warning .el-tag__close:hover { + color: #fff; + background-color: #ebb563; +} + +.el-tag--dark.el-tag--danger { + background-color: #f56c6c; + border-color: #f56c6c; + color: #fff; +} + +.el-tag--dark.el-tag--danger.is-hit { + border-color: #f56c6c; +} + +.el-tag--dark.el-tag--danger .el-tag__close { + color: #fff; +} + +.el-tag--dark.el-tag--danger .el-tag__close:hover { + color: #fff; + background-color: #f78989; +} + +.el-tag--plain { + background-color: #fff; + border-color: rgb(153, 219, 220); + color: #00a5a7; +} + +.el-tag--plain.is-hit { + border-color: #00a5a7; +} + +.el-tag--plain .el-tag__close { + color: #00a5a7; +} + +.el-tag--plain .el-tag__close:hover { + color: #fff; + background-color: #00a5a7; +} + +.el-tag--plain.el-tag--info { + background-color: #fff; + border-color: #d3d4d6; + color: #909399; +} + +.el-tag--plain.el-tag--info.is-hit { + border-color: #909399; +} + +.el-tag--plain.el-tag--info .el-tag__close { + color: #909399; +} + +.el-tag--plain.el-tag--info .el-tag__close:hover { + color: #fff; + background-color: #909399; +} + +.el-tag--plain.el-tag--success { + background-color: #fff; + border-color: #c2e7b0; + color: #67c23a; +} + +.el-tag--plain.el-tag--success.is-hit { + border-color: #67c23a; +} + +.el-tag--plain.el-tag--success .el-tag__close { + color: #67c23a; +} + +.el-tag--plain.el-tag--success .el-tag__close:hover { + color: #fff; + background-color: #67c23a; +} + +.el-tag--plain.el-tag--warning { + background-color: #fff; + border-color: #f5dab1; + color: #e6a23c; +} + +.el-tag--plain.el-tag--warning.is-hit { + border-color: #e6a23c; +} + +.el-tag--plain.el-tag--warning .el-tag__close { + color: #e6a23c; +} + +.el-tag--plain.el-tag--warning .el-tag__close:hover { + color: #fff; + background-color: #e6a23c; +} + +.el-tag--plain.el-tag--danger { + background-color: #fff; + border-color: #fbc4c4; + color: #f56c6c; +} + +.el-tag--plain.el-tag--danger.is-hit { + border-color: #f56c6c; +} + +.el-tag--plain.el-tag--danger .el-tag__close { + color: #f56c6c; +} + +.el-tag--plain.el-tag--danger .el-tag__close:hover { + color: #fff; + background-color: #f56c6c; +} + +.el-tag--medium { + height: 28px; + line-height: 26px; +} + +.el-tag--medium .el-icon-close { + -webkit-transform: scale(0.8); + transform: scale(0.8); +} + +.el-tag--small { + height: 24px; + padding: 0 8px; + line-height: 22px; +} + +.el-tag--small .el-icon-close { + -webkit-transform: scale(0.8); + transform: scale(0.8); +} + +.el-tag--mini { + height: 20px; + padding: 0 5px; + line-height: 19px; +} + +.el-tag--mini .el-icon-close { + margin-left: -3px; + -webkit-transform: scale(0.7); + transform: scale(0.7); +} + +.el-cascader { + display: inline-block; + position: relative; + font-size: 14px; + line-height: 40px; +} + +.el-cascader:not(.is-disabled):hover .el-input__inner { + cursor: pointer; + border-color: #c0c4cc; +} + +.el-cascader .el-input .el-input__inner:focus, +.el-cascader .el-input.is-focus .el-input__inner { + border-color: #00a5a7; +} + +.el-cascader .el-input { + cursor: pointer; +} + +.el-cascader .el-input .el-input__inner { + text-overflow: ellipsis; +} + +.el-cascader .el-input .el-icon-arrow-down { + -webkit-transition: -webkit-transform 0.3s; + transition: -webkit-transform 0.3s; + transition: transform 0.3s; + transition: transform 0.3s, -webkit-transform 0.3s; + font-size: 14px; +} + +.el-cascader .el-input .el-icon-arrow-down.is-reverse { + -webkit-transform: rotateZ(180deg); + transform: rotateZ(180deg); +} + +.el-cascader .el-input .el-icon-circle-close:hover { + color: #909399; +} + +.el-cascader--medium { + font-size: 14px; + line-height: 36px; +} + +.el-cascader--small { + font-size: 13px; + line-height: 32px; +} + +.el-cascader--mini { + font-size: 12px; + line-height: 28px; +} + +.el-cascader.is-disabled .el-cascader__label { + z-index: 2; + color: #c0c4cc; +} + +.el-cascader__dropdown { + margin: 5px 0; + font-size: 14px; + background: #fff; + border: 1px solid #e4e7ed; + border-radius: 4px; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-cascader__tags { + position: absolute; + left: 0; + right: 30px; + top: 50%; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -ms-flex-wrap: wrap; + flex-wrap: wrap; + line-height: normal; + text-align: left; + box-sizing: border-box; +} + +.el-cascader__tags .el-tag { + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + max-width: 100%; + margin: 2px 0 2px 6px; + text-overflow: ellipsis; + background: #f0f2f5; +} + +.el-cascader__tags .el-tag:not(.is-hit) { + border-color: transparent; +} + +.el-cascader__tags .el-tag > span { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + overflow: hidden; + text-overflow: ellipsis; +} + +.el-cascader__tags .el-tag .el-icon-close { + -webkit-box-flex: 0; + -ms-flex: none; + flex: none; + background-color: #c0c4cc; + color: #fff; +} + +.el-cascader__tags .el-tag .el-icon-close:hover { + background-color: #909399; +} + +.el-cascader__suggestion-panel { + border-radius: 4px; +} + +.el-cascader__suggestion-list { + max-height: 204px; + margin: 0; + padding: 6px 0; + font-size: 14px; + color: #606266; + text-align: center; +} + +.el-cascader__suggestion-item { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-pack: justify; + -ms-flex-pack: justify; + justify-content: space-between; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + height: 34px; + padding: 0 15px; + text-align: left; + outline: 0; + cursor: pointer; +} + +.el-cascader__suggestion-item:focus, +.el-cascader__suggestion-item:hover { + background: #f5f7fa; +} + +.el-cascader__suggestion-item.is-checked { + color: #00a5a7; + font-weight: 700; +} + +.el-cascader__suggestion-item > span { + margin-right: 10px; +} + +.el-cascader__empty-text { + margin: 10px 0; + color: #c0c4cc; +} + +.el-cascader__search-input { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + height: 24px; + min-width: 60px; + margin: 2px 0 2px 15px; + padding: 0; + color: #606266; + border: none; + outline: 0; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-cascader__search-input::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-cascader__search-input:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-cascader__search-input::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-cascader__search-input::placeholder { + color: #c0c4cc; +} + +.el-color-predefine { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + font-size: 12px; + margin-top: 8px; + width: 280px; +} + +.el-color-predefine__colors { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + -ms-flex-wrap: wrap; + flex-wrap: wrap; +} + +.el-color-predefine__color-selector { + margin: 0 0 8px 8px; + width: 20px; + height: 20px; + border-radius: 4px; + cursor: pointer; +} + +.el-color-predefine__color-selector:nth-child(10n + 1) { + margin-left: 0; +} + +.el-color-predefine__color-selector.selected { + -webkit-box-shadow: 0 0 3px 2px #00a5a7; + box-shadow: 0 0 3px 2px #00a5a7; +} + +.el-color-predefine__color-selector > div { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + height: 100%; + border-radius: 3px; +} + +.el-color-predefine__color-selector.is-alpha { + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAwAAAAMCAIAAADZF8uwAAAAGUlEQVQYV2M4gwH+YwCGIasIUwhT25BVBADtzYNYrHvv4gAAAABJRU5ErkJggg==); +} + +.el-color-hue-slider { + position: relative; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 280px; + height: 12px; + background-color: red; + padding: 0 2px; +} + +.el-color-hue-slider__bar { + position: relative; + background: -webkit-gradient( + linear, + left top, + right top, + from(red), + color-stop(17%, #ff0), + color-stop(33%, #0f0), + color-stop(50%, #0ff), + color-stop(67%, #00f), + color-stop(83%, #f0f), + to(red) + ); + background: linear-gradient( + to right, + red 0, + #ff0 17%, + #0f0 33%, + #0ff 50%, + #00f 67%, + #f0f 83%, + red 100% + ); + height: 100%; +} + +.el-color-hue-slider__thumb { + position: absolute; + cursor: pointer; + -webkit-box-sizing: border-box; + box-sizing: border-box; + left: 0; + top: 0; + width: 4px; + height: 100%; + border-radius: 1px; + background: #fff; + border: 1px solid #f0f0f0; + -webkit-box-shadow: 0 0 2px rgba(0, 0, 0, 0.6); + box-shadow: 0 0 2px rgba(0, 0, 0, 0.6); + z-index: 1; +} + +.el-color-hue-slider.is-vertical { + width: 12px; + height: 180px; + padding: 2px 0; +} + +.el-color-hue-slider.is-vertical .el-color-hue-slider__bar { + background: -webkit-gradient( + linear, + left top, + left bottom, + from(red), + color-stop(17%, #ff0), + color-stop(33%, #0f0), + color-stop(50%, #0ff), + color-stop(67%, #00f), + color-stop(83%, #f0f), + to(red) + ); + background: linear-gradient( + to bottom, + red 0, + #ff0 17%, + #0f0 33%, + #0ff 50%, + #00f 67%, + #f0f 83%, + red 100% + ); +} + +.el-color-hue-slider.is-vertical .el-color-hue-slider__thumb { + left: 0; + top: 0; + width: 100%; + height: 4px; +} + +.el-color-svpanel { + position: relative; + width: 280px; + height: 180px; +} + +.el-color-svpanel__black, +.el-color-svpanel__white { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; +} + +.el-color-svpanel__white { + background: -webkit-gradient( + linear, + left top, + right top, + from(#fff), + to(rgba(255, 255, 255, 0)) + ); + background: linear-gradient(to right, #fff, rgba(255, 255, 255, 0)); +} + +.el-color-svpanel__black { + background: -webkit-gradient(linear, left bottom, left top, from(#000), to(rgba(0, 0, 0, 0))); + background: linear-gradient(to top, #000, rgba(0, 0, 0, 0)); +} + +.el-color-svpanel__cursor { + position: absolute; +} + +.el-color-svpanel__cursor > div { + cursor: head; + width: 4px; + height: 4px; + -webkit-box-shadow: 0 0 0 1.5px #fff, inset 0 0 1px 1px rgba(0, 0, 0, 0.3), + 0 0 1px 2px rgba(0, 0, 0, 0.4); + box-shadow: 0 0 0 1.5px #fff, inset 0 0 1px 1px rgba(0, 0, 0, 0.3), + 0 0 1px 2px rgba(0, 0, 0, 0.4); + border-radius: 50%; + -webkit-transform: translate(-2px, -2px); + transform: translate(-2px, -2px); +} + +.el-color-alpha-slider { + position: relative; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 280px; + height: 12px; + background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAwAAAAMCAIAAADZF8uwAAAAGUlEQVQYV2M4gwH+YwCGIasIUwhT25BVBADtzYNYrHvv4gAAAABJRU5ErkJggg==); +} + +.el-color-alpha-slider__bar { + position: relative; + background: -webkit-gradient( + linear, + left top, + right top, + from(rgba(255, 255, 255, 0)), + to(white) + ); + background: linear-gradient(to right, rgba(255, 255, 255, 0) 0, #fff 100%); + height: 100%; +} + +.el-color-alpha-slider__thumb { + position: absolute; + cursor: pointer; + -webkit-box-sizing: border-box; + box-sizing: border-box; + left: 0; + top: 0; + width: 4px; + height: 100%; + border-radius: 1px; + background: #fff; + border: 1px solid #f0f0f0; + -webkit-box-shadow: 0 0 2px rgba(0, 0, 0, 0.6); + box-shadow: 0 0 2px rgba(0, 0, 0, 0.6); + z-index: 1; +} + +.el-color-alpha-slider.is-vertical { + width: 20px; + height: 180px; +} + +.el-color-alpha-slider.is-vertical .el-color-alpha-slider__bar { + background: -webkit-gradient( + linear, + left top, + left bottom, + from(rgba(255, 255, 255, 0)), + to(white) + ); + background: linear-gradient(to bottom, rgba(255, 255, 255, 0) 0, #fff 100%); +} + +.el-color-alpha-slider.is-vertical .el-color-alpha-slider__thumb { + left: 0; + top: 0; + width: 100%; + height: 4px; +} + +.el-color-dropdown { + width: 300px; +} + +.el-color-dropdown__main-wrapper { + margin-bottom: 6px; +} + +.el-color-dropdown__main-wrapper::after { + content: ''; + display: table; + clear: both; +} + +.el-color-dropdown__btns { + margin-top: 6px; + text-align: right; +} + +.el-color-dropdown__value { + float: left; + line-height: 26px; + font-size: 12px; + color: #000; + width: 160px; +} + +.el-color-dropdown__btn { + border: 1px solid #dcdcdc; + color: #333; + line-height: 24px; + border-radius: 2px; + padding: 0 20px; + cursor: pointer; + background-color: transparent; + outline: 0; + font-size: 12px; +} + +.el-color-dropdown__btn[disabled] { + color: #ccc; + cursor: not-allowed; +} + +.el-color-dropdown__btn:hover { + color: #00a5a7; + border-color: #00a5a7; +} + +.el-color-dropdown__link-btn { + cursor: pointer; + color: #00a5a7; + text-decoration: none; + padding: 15px; + font-size: 12px; +} + +.el-color-dropdown__link-btn:hover { + color: tint(primary, 20%); +} + +.el-color-picker { + display: inline-block; + position: relative; + line-height: normal; + height: 40px; +} + +.el-color-picker.is-disabled .el-color-picker__trigger { + cursor: not-allowed; +} + +.el-color-picker--medium { + height: 36px; +} + +.el-color-picker--medium .el-color-picker__trigger { + height: 36px; + width: 36px; +} + +.el-color-picker--medium .el-color-picker__mask { + height: 34px; + width: 34px; +} + +.el-color-picker--small { + height: 32px; +} + +.el-color-picker--small .el-color-picker__trigger { + height: 32px; + width: 32px; +} + +.el-color-picker--small .el-color-picker__mask { + height: 30px; + width: 30px; +} + +.el-color-picker--small .el-color-picker__empty, +.el-color-picker--small .el-color-picker__icon { + -webkit-transform: translate3d(-50%, -50%, 0) scale(0.8); + transform: translate3d(-50%, -50%, 0) scale(0.8); +} + +.el-color-picker--mini { + height: 28px; +} + +.el-color-picker--mini .el-color-picker__trigger { + height: 28px; + width: 28px; +} + +.el-color-picker--mini .el-color-picker__mask { + height: 26px; + width: 26px; +} + +.el-color-picker--mini .el-color-picker__empty, +.el-color-picker--mini .el-color-picker__icon { + -webkit-transform: translate3d(-50%, -50%, 0) scale(0.8); + transform: translate3d(-50%, -50%, 0) scale(0.8); +} + +.el-color-picker__mask { + height: 38px; + width: 38px; + border-radius: 4px; + position: absolute; + top: 1px; + left: 1px; + z-index: 1; + cursor: not-allowed; + background-color: rgba(255, 255, 255, 0.7); +} + +.el-color-picker__trigger { + display: inline-block; + -webkit-box-sizing: border-box; + box-sizing: border-box; + height: 40px; + width: 40px; + padding: 4px; + border: 1px solid #e6e6e6; + border-radius: 4px; + font-size: 0; + position: relative; + cursor: pointer; +} + +.el-color-picker__color { + position: relative; + display: block; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border: 1px solid #999; + border-radius: 2px; + width: 100%; + height: 100%; + text-align: center; +} + +.el-color-picker__color.is-alpha { + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAwAAAAMCAIAAADZF8uwAAAAGUlEQVQYV2M4gwH+YwCGIasIUwhT25BVBADtzYNYrHvv4gAAAABJRU5ErkJggg==); +} + +.el-color-picker__color-inner { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +.el-color-picker__empty, +.el-color-picker__icon { + top: 50%; + left: 50%; + font-size: 12px; + position: absolute; +} + +.el-color-picker__empty { + color: #999; + -webkit-transform: translate3d(-50%, -50%, 0); + transform: translate3d(-50%, -50%, 0); +} + +.el-color-picker__icon { + display: inline-block; + width: 100%; + -webkit-transform: translate3d(-50%, -50%, 0); + transform: translate3d(-50%, -50%, 0); + color: #fff; + text-align: center; +} + +.el-color-picker__panel { + position: absolute; + z-index: 10; + padding: 6px; + -webkit-box-sizing: content-box; + box-sizing: content-box; + background-color: #fff; + border: 1px solid #ebeef5; + border-radius: 4px; + -webkit-box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 12px 0 rgba(0, 0, 0, 0.1); +} + +.el-textarea { + position: relative; + display: inline-block; + width: 100%; + vertical-align: bottom; + font-size: 14px; +} + +.empty-content .el-textarea__inner { + border: none; + padding: 0; +} + +.el-textarea__inner { + display: block; + resize: vertical; + padding: 5px 15px; + line-height: 1.5; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 100%; + font-size: inherit; + color: #606266; + background-color: #fff; + background-image: none; + border: 1px solid #dcdfe6; + border-radius: 4px; + -webkit-transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); +} + +.el-textarea__inner::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-textarea__inner:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-textarea__inner::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-textarea__inner::placeholder { + color: #c0c4cc; +} + +.el-textarea__inner:hover { + border-color: #c0c4cc; +} + +.el-textarea__inner:focus { + outline: 0; + border-color: #00a5a7; +} + +.el-textarea .el-input__count { + color: #909399; + background: #fff; + position: absolute; + font-size: 12px; + bottom: 5px; + right: 10px; +} + +.el-textarea.is-disabled .el-textarea__inner { + background-color: #f5f7fa; + border-color: #e4e7ed; + color: #c0c4cc; + cursor: not-allowed; +} + +.el-textarea.is-disabled .el-textarea__inner::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-textarea.is-disabled .el-textarea__inner:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-textarea.is-disabled .el-textarea__inner::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-textarea.is-disabled .el-textarea__inner::placeholder { + color: #c0c4cc; +} + +.el-textarea.is-exceed .el-textarea__inner { + border-color: #f56c6c; +} + +.el-textarea.is-exceed .el-input__count { + color: #f56c6c; +} + +.el-input { + position: relative; + font-size: 14px; + display: inline-block; + width: 100%; +} + +.el-input::-webkit-scrollbar { + z-index: 11; + width: 6px; +} + +.el-input::-webkit-scrollbar:horizontal { + height: 6px; +} + +.el-input::-webkit-scrollbar-thumb { + border-radius: 5px; + width: 6px; + background: #b4bccc; +} + +.el-input::-webkit-scrollbar-corner { + background: #fff; +} + +.el-input::-webkit-scrollbar-track { + background: #fff; +} + +.el-input::-webkit-scrollbar-track-piece { + background: #fff; + width: 6px; +} + +.el-input .el-input__clear { + color: #c0c4cc; + font-size: 14px; + cursor: pointer; + -webkit-transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); +} + +.el-input .el-input__clear:hover { + color: #909399; +} + +.el-input .el-input__count { + height: 100%; + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + color: #909399; + font-size: 12px; +} + +.el-input .el-input__count .el-input__count-inner { + background: #fff; + line-height: initial; + display: inline-block; + padding: 0 5px; +} + +.el-input__inner { + -webkit-appearance: none; + background-color: #fff; + background-image: none; + border-radius: 1px; + border: 1px solid #dcdfe6; + -webkit-box-sizing: border-box; + box-sizing: border-box; + color: #606266; + display: inline-block; + font-size: inherit; + height: 32px; + line-height: 32px; + outline: 0; + padding: 0 15px; + -webkit-transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: border-color 0.2s cubic-bezier(0.645, 0.045, 0.355, 1); + width: 100%; +} + +.el-button, +.el-transfer-panel { + -webkit-box-sizing: border-box; +} + +.el-input__prefix, +.el-input__suffix { + position: absolute; + top: 0; + -webkit-transition: all 0.3s; + height: 100%; + color: #c0c4cc; + text-align: center; +} + +.el-input__inner::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-input__inner:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-input__inner::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-input__inner::placeholder { + color: #c0c4cc; +} + +.el-input__inner:hover { + border-color: #c0c4cc; +} + +.el-input.is-active .el-input__inner, +.el-input__inner:focus { + border-color: #00a5a7; + outline: 0; +} + +.el-input__suffix { + right: 5px; + transition: all 0.3s; +} + +.el-input__suffix-inner { + pointer-events: all; +} + +.el-input__prefix { + left: 5px; + transition: all 0.3s; +} + +.el-input__icon { + height: 100%; + width: 25px; + text-align: center; + -webkit-transition: all 0.3s; + transition: all 0.3s; + line-height: 32px; +} + +.el-input__icon:after { + content: ''; + height: 100%; + width: 0; + display: inline-block; + vertical-align: middle; +} + +.el-input__validateIcon { + pointer-events: none; +} + +.el-input.is-disabled .el-input__inner { + background-color: #f5f7fa; + border-color: #e4e7ed; + color: #c0c4cc; + cursor: not-allowed; +} + +.el-input.is-disabled .el-input__inner::-webkit-input-placeholder { + color: #c0c4cc; +} + +.el-input.is-disabled .el-input__inner:-ms-input-placeholder { + color: #c0c4cc; +} + +.el-input.is-disabled .el-input__inner::-ms-input-placeholder { + color: #c0c4cc; +} + +.el-input.is-disabled .el-input__inner::placeholder { + color: #c0c4cc; +} + +.el-input.is-disabled .el-input__icon { + cursor: not-allowed; +} + +.el-input.is-exceed .el-input__inner { + border-color: #f56c6c; +} + +.el-input.is-exceed .el-input__suffix .el-input__count { + color: #f56c6c; +} + +.el-input--suffix .el-input__inner { + padding-right: 30px; +} + +.el-input--prefix .el-input__inner { + padding-left: 30px; +} + +.el-input--medium { + font-size: 14px; +} + +.el-input--medium .el-input__inner { + height: 36px; + line-height: 36px; +} + +.el-input--medium .el-input__icon { + line-height: 36px; +} + +.el-input--small { + font-size: 13px; +} + +.el-input--small .el-input__inner { + height: 32px; + line-height: 32px; +} + +.el-input--small .el-input__icon { + line-height: 32px; +} + +.el-input--mini { + font-size: 12px; +} + +.el-input--mini .el-input__inner { + height: 28px; + line-height: 28px; +} + +.el-input--mini .el-input__icon { + line-height: 28px; +} + +.el-input-group { + line-height: normal; + display: inline-table; + width: 100%; + border-collapse: separate; + border-spacing: 0; +} + +.el-input-group > .el-input__inner { + vertical-align: middle; + display: table-cell; +} + +.el-input-group__append, +.el-input-group__prepend { + background-color: #f5f7fa; + color: #909399; + vertical-align: middle; + display: table-cell; + position: relative; + border: 1px solid #dcdfe6; + border-radius: 4px; + padding: 0 20px; + width: 1px; + white-space: nowrap; +} + +.el-input-group--prepend .el-input__inner, +.el-input-group__append { + border-top-left-radius: 0; + border-bottom-left-radius: 0; +} + +.el-input-group--append .el-input__inner, +.el-input-group__prepend { + border-top-right-radius: 0; + border-bottom-right-radius: 0; +} + +.el-input-group__append:focus, +.el-input-group__prepend:focus { + outline: 0; +} + +.el-input-group__append .el-button, +.el-input-group__append .el-select, +.el-input-group__prepend .el-button, +.el-input-group__prepend .el-select { + display: inline-block; + margin: -10px -20px; +} + +.el-input-group__append button.el-button, +.el-input-group__append div.el-select .el-input__inner, +.el-input-group__append div.el-select:hover .el-input__inner, +.el-input-group__prepend button.el-button, +.el-input-group__prepend div.el-select .el-input__inner, +.el-input-group__prepend div.el-select:hover .el-input__inner { + border-color: transparent; + background-color: transparent; + color: inherit; + border-top: 0; + border-bottom: 0; +} + +.el-input-group__append .el-button, +.el-input-group__append .el-input, +.el-input-group__prepend .el-button, +.el-input-group__prepend .el-input { + font-size: inherit; +} + +.el-input-group__prepend { + border-right: 0; +} + +.el-input-group__append { + border-left: 0; +} + +.el-input-group--append .el-select .el-input.is-focus .el-input__inner, +.el-input-group--prepend .el-select .el-input.is-focus .el-input__inner { + border-color: transparent; +} + +.el-input__inner::-ms-clear { + display: none; + width: 0; + height: 0; +} + +.el-button { + display: inline-block; + line-height: 1; + white-space: nowrap; + cursor: pointer; + background: #fff; + border: 1px solid #dcdfe6; + color: #606266; + -webkit-appearance: none; + text-align: center; + box-sizing: border-box; + outline: 0; + margin: 0; + -webkit-transition: 0.1s; + transition: 0.1s; + font-weight: 500; + padding: 12px 20px; + font-size: 14px; + border-radius: 4px; +} + +.el-button + .el-button { + margin-left: 10px; +} + +.el-button:focus, +.el-button:hover { + color: #00a5a7; + border-color: rgb(179, 228, 229); + background-color: rgb(230, 246, 246); +} + +.el-button:active { + color: rgb(0, 149, 150); + border-color: rgb(0, 149, 150); + outline: 0; +} + +.el-button::-moz-focus-inner { + border: 0; +} + +.el-button [class*='el-icon-'] + span { + margin-left: 5px; +} + +.el-button.is-plain:focus, +.el-button.is-plain:hover { + background: #fff; + border-color: #00a5a7; + color: #00a5a7; +} + +.el-button.is-active, +.el-button.is-plain:active { + color: rgb(0, 149, 150); + border-color: rgb(0, 149, 150); +} + +.el-button.is-plain:active { + background: #fff; + outline: 0; +} + +.el-button.is-disabled, +.el-button.is-disabled:focus, +.el-button.is-disabled:hover { + color: #c0c4cc; + cursor: not-allowed; + background-image: none; + background-color: #fff; + border-color: #ebeef5; +} + +.el-link, +.el-transfer-panel__filter .el-icon-circle-close { + cursor: pointer; +} + +.el-button.is-disabled.el-button--text { + background-color: transparent; +} + +.el-button.is-disabled.is-plain, +.el-button.is-disabled.is-plain:focus, +.el-button.is-disabled.is-plain:hover { + background-color: #fff; + border-color: #ebeef5; + color: #c0c4cc; +} + +.el-button.is-loading { + position: relative; + pointer-events: none; +} + +.el-button.is-loading:before { + pointer-events: none; + content: ''; + position: absolute; + left: -1px; + top: -1px; + right: -1px; + bottom: -1px; + border-radius: inherit; + background-color: rgba(255, 255, 255, 0.35); +} + +.el-button.is-round { + border-radius: 20px; + padding: 12px 23px; +} + +.el-button.is-circle { + border-radius: 50%; + padding: 12px; +} + +.el-button--primary { + color: #fff; + background-color: #00a5a7; + border-color: #00a5a7; +} + +.el-button--primary:focus, +.el-button--primary:hover { + background: rgb(51, 183, 185); + border-color: rgb(51, 183, 185); + color: #fff; +} + +.el-button--primary.is-active, +.el-button--primary:active { + background: rgb(0, 149, 150); + border-color: rgb(0, 149, 150); + color: #fff; +} + +.el-button--primary:active { + outline: 0; +} + +.el-button--primary.is-disabled, +.el-button--primary.is-disabled:active, +.el-button--primary.is-disabled:focus, +.el-button--primary.is-disabled:hover { + color: #fff; + background-color: rgb(128, 210, 211); + border-color: rgb(128, 210, 211); +} + +.el-button--primary.is-plain { + color: #00a5a7; + background: rgb(230, 246, 246); + border-color: rgb(153, 219, 220); +} + +.el-button--primary.is-plain:focus, +.el-button--primary.is-plain:hover { + background: #00a5a7; + border-color: #00a5a7; + color: #fff; +} + +.el-button--primary.is-plain:active { + background: rgb(0, 149, 150); + border-color: rgb(0, 149, 150); + color: #fff; + outline: 0; +} + +.el-button--primary.is-plain.is-disabled, +.el-button--primary.is-plain.is-disabled:active, +.el-button--primary.is-plain.is-disabled:focus, +.el-button--primary.is-plain.is-disabled:hover { + color: rgb(102, 201, 202); + background-color: rgb(230, 246, 246); + border-color: rgb(204, 237, 237); +} + +.el-button--success { + color: #fff; + background-color: #67c23a; + border-color: #67c23a; +} + +.el-button--success:focus, +.el-button--success:hover { + background: #85ce61; + border-color: #85ce61; + color: #fff; +} + +.el-button--success.is-active, +.el-button--success:active { + background: #5daf34; + border-color: #5daf34; + color: #fff; +} + +.el-button--success:active { + outline: 0; +} + +.el-button--success.is-disabled, +.el-button--success.is-disabled:active, +.el-button--success.is-disabled:focus, +.el-button--success.is-disabled:hover { + color: #fff; + background-color: #b3e19d; + border-color: #b3e19d; +} + +.el-button--success.is-plain { + color: #67c23a; + background: #f0f9eb; + border-color: #c2e7b0; +} + +.el-button--success.is-plain:focus, +.el-button--success.is-plain:hover { + background: #67c23a; + border-color: #67c23a; + color: #fff; +} + +.el-button--success.is-plain:active { + background: #5daf34; + border-color: #5daf34; + color: #fff; + outline: 0; +} + +.el-button--success.is-plain.is-disabled, +.el-button--success.is-plain.is-disabled:active, +.el-button--success.is-plain.is-disabled:focus, +.el-button--success.is-plain.is-disabled:hover { + color: #a4da89; + background-color: #f0f9eb; + border-color: #e1f3d8; +} + +.el-button--warning { + color: #fff; + background-color: #e6a23c; + border-color: #e6a23c; +} + +.el-button--warning:focus, +.el-button--warning:hover { + background: #ebb563; + border-color: #ebb563; + color: #fff; +} + +.el-button--warning.is-active, +.el-button--warning:active { + background: #cf9236; + border-color: #cf9236; + color: #fff; +} + +.el-button--warning:active { + outline: 0; +} + +.el-button--warning.is-disabled, +.el-button--warning.is-disabled:active, +.el-button--warning.is-disabled:focus, +.el-button--warning.is-disabled:hover { + color: #fff; + background-color: #f3d19e; + border-color: #f3d19e; +} + +.el-button--warning.is-plain { + color: #e6a23c; + background: #fdf6ec; + border-color: #f5dab1; +} + +.el-button--warning.is-plain:focus, +.el-button--warning.is-plain:hover { + background: #e6a23c; + border-color: #e6a23c; + color: #fff; +} + +.el-button--warning.is-plain:active { + background: #cf9236; + border-color: #cf9236; + color: #fff; + outline: 0; +} + +.el-button--warning.is-plain.is-disabled, +.el-button--warning.is-plain.is-disabled:active, +.el-button--warning.is-plain.is-disabled:focus, +.el-button--warning.is-plain.is-disabled:hover { + color: #f0c78a; + background-color: #fdf6ec; + border-color: #faecd8; +} + +.el-button--danger { + color: #fff; + background-color: #f56c6c; + border-color: #f56c6c; +} + +.el-button--danger:focus, +.el-button--danger:hover { + background: #f78989; + border-color: #f78989; + color: #fff; +} + +.el-button--danger.is-active, +.el-button--danger:active { + background: #dd6161; + border-color: #dd6161; + color: #fff; +} + +.el-button--danger:active { + outline: 0; +} + +.el-button--danger.is-disabled, +.el-button--danger.is-disabled:active, +.el-button--danger.is-disabled:focus, +.el-button--danger.is-disabled:hover { + color: #fff; + background-color: #fab6b6; + border-color: #fab6b6; +} + +.el-button--danger.is-plain { + color: #f56c6c; + background: #fef0f0; + border-color: #fbc4c4; +} + +.el-button--danger.is-plain:focus, +.el-button--danger.is-plain:hover { + background: #f56c6c; + border-color: #f56c6c; + color: #fff; +} + +.el-button--danger.is-plain:active { + background: #dd6161; + border-color: #dd6161; + color: #fff; + outline: 0; +} + +.el-button--danger.is-plain.is-disabled, +.el-button--danger.is-plain.is-disabled:active, +.el-button--danger.is-plain.is-disabled:focus, +.el-button--danger.is-plain.is-disabled:hover { + color: #f9a7a7; + background-color: #fef0f0; + border-color: #fde2e2; +} + +.el-button--info { + color: #fff; + background-color: #909399; + border-color: #909399; +} + +.el-button--info:focus, +.el-button--info:hover { + background: #a6a9ad; + border-color: #a6a9ad; + color: #fff; +} + +.el-button--info.is-active, +.el-button--info:active { + background: #82848a; + border-color: #82848a; + color: #fff; +} + +.el-button--info:active { + outline: 0; +} + +.el-button--info.is-disabled, +.el-button--info.is-disabled:active, +.el-button--info.is-disabled:focus, +.el-button--info.is-disabled:hover { + color: #fff; + background-color: #c8c9cc; + border-color: #c8c9cc; +} + +.el-button--info.is-plain { + color: #909399; + background: #f4f4f5; + border-color: #d3d4d6; +} + +.el-button--info.is-plain:focus, +.el-button--info.is-plain:hover { + background: #909399; + border-color: #909399; + color: #fff; +} + +.el-button--info.is-plain:active { + background: #82848a; + border-color: #82848a; + color: #fff; + outline: 0; +} + +.el-button--info.is-plain.is-disabled, +.el-button--info.is-plain.is-disabled:active, +.el-button--info.is-plain.is-disabled:focus, +.el-button--info.is-plain.is-disabled:hover { + color: #bcbec2; + background-color: #f4f4f5; + border-color: #e9e9eb; +} + +.el-button--text, +.el-button--text.is-disabled, +.el-button--text.is-disabled:focus, +.el-button--text.is-disabled:hover, +.el-button--text:active { + border-color: transparent; +} + +.el-button--medium { + padding: 10px 20px; + font-size: 14px; + border-radius: 4px; +} + +.el-button--mini, +.el-button--small { + font-size: 12px; + border-radius: 3px; +} + +.el-button--medium.is-round { + padding: 10px 20px; +} + +.el-button--medium.is-circle { + padding: 10px; +} + +.el-button--small, +.el-button--small.is-round { + padding: 9px 15px; +} + +.el-button--small.is-circle { + padding: 9px; +} + +.el-button--mini, +.el-button--mini.is-round { + padding: 7px 15px; +} + +.el-button--mini.is-circle { + padding: 7px; +} + +.el-button--text { + color: #00a5a7; + background: 0 0; + padding-left: 0; + padding-right: 0; +} + +.el-button--text:focus, +.el-button--text:hover { + color: rgb(51, 183, 185); + border-color: transparent; + background-color: transparent; +} + +.el-button--text:active { + color: rgb(0, 149, 150); + background-color: transparent; +} + +.el-button-group { + display: inline-block; + vertical-align: middle; +} + +.el-button-group::after, +.el-button-group::before { + display: table; + content: ''; +} + +.el-button-group::after { + clear: both; +} + +.el-button-group > .el-button { + float: left; + position: relative; +} + +.el-button-group > .el-button + .el-button { + margin-left: 0; +} + +.el-button-group > .el-button.is-disabled { + z-index: 1; +} + +.el-button-group > .el-button:first-child { + border-top-right-radius: 0; + border-bottom-right-radius: 0; +} + +.el-button-group > .el-button:last-child { + border-top-left-radius: 0; + border-bottom-left-radius: 0; +} + +.el-button-group > .el-button:first-child:last-child { + border-radius: 4px; +} + +.el-button-group > .el-button:first-child:last-child.is-round { + border-radius: 20px; +} + +.el-button-group > .el-button:first-child:last-child.is-circle { + border-radius: 50%; +} + +.el-button-group > .el-button:not(:first-child):not(:last-child) { + border-radius: 0; +} + +.el-button-group > .el-button:not(:last-child) { + margin-right: -1px; +} + +.el-button-group > .el-button.is-active, +.el-button-group > .el-button:active, +.el-button-group > .el-button:focus, +.el-button-group > .el-button:hover { + z-index: 1; +} + +.el-button-group > .el-dropdown > .el-button { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--primary:first-child { + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--primary:last-child { + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--primary:not(:first-child):not(:last-child) { + border-left-color: rgba(255, 255, 255, 0.5); + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--success:first-child { + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--success:last-child { + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--success:not(:first-child):not(:last-child) { + border-left-color: rgba(255, 255, 255, 0.5); + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--warning:first-child { + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--warning:last-child { + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--warning:not(:first-child):not(:last-child) { + border-left-color: rgba(255, 255, 255, 0.5); + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--danger:first-child { + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--danger:last-child { + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--danger:not(:first-child):not(:last-child) { + border-left-color: rgba(255, 255, 255, 0.5); + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--info:first-child { + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--info:last-child { + border-left-color: rgba(255, 255, 255, 0.5); +} + +.el-button-group .el-button--info:not(:first-child):not(:last-child) { + border-left-color: rgba(255, 255, 255, 0.5); + border-right-color: rgba(255, 255, 255, 0.5); +} + +.el-transfer { + font-size: 14px; +} + +.el-transfer__buttons { + display: inline-block; + vertical-align: middle; + padding: 0 30px; +} + +.el-transfer__button { + display: block; + margin: 0 auto; + padding: 10px; + border-radius: 50%; + color: #fff; + background-color: #00a5a7; + font-size: 0; +} + +.el-transfer__button.is-with-texts { + border-radius: 4px; +} + +.el-transfer__button.is-disabled, +.el-transfer__button.is-disabled:hover { + border: 1px solid #dcdfe6; + background-color: #f5f7fa; + color: #c0c4cc; +} + +.el-transfer__button:first-child { + margin-bottom: 10px; +} + +.el-transfer__button:nth-child(2) { + margin: 0; +} + +.el-transfer__button i, +.el-transfer__button span { + font-size: 14px; +} + +.el-transfer__button [class*='el-icon-'] + span { + margin-left: 0; +} + +.el-transfer-panel { + border: 1px solid #ebeef5; + border-radius: 4px; + overflow: hidden; + background: #fff; + display: inline-block; + vertical-align: middle; + width: 200px; + max-height: 100%; + box-sizing: border-box; + position: relative; +} + +.el-transfer-panel__body { + height: 246px; +} + +.el-transfer-panel__body.is-with-footer { + padding-bottom: 40px; +} + +.el-transfer-panel__list { + margin: 0; + padding: 6px 0; + list-style: none; + height: 246px; + overflow: auto; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-transfer-panel__list.is-filterable { + height: 194px; + padding-top: 0; +} + +.el-transfer-panel__item { + height: 30px; + line-height: 30px; + padding-left: 15px; + display: block; +} + +.el-transfer-panel__item + .el-transfer-panel__item { + margin-left: 0; + display: block !important; +} + +.el-transfer-panel__item.el-checkbox { + color: #606266; +} + +.el-transfer-panel__item:hover { + color: #00a5a7; +} + +.el-transfer-panel__item.el-checkbox .el-checkbox__label { + width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + display: block; + -webkit-box-sizing: border-box; + box-sizing: border-box; + padding-left: 24px; + line-height: 30px; +} + +.el-transfer-panel__item .el-checkbox__input { + position: absolute; + top: 8px; +} + +.el-transfer-panel__filter { + text-align: center; + margin: 15px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + display: block; + width: auto; +} + +.el-transfer-panel__filter .el-input__inner { + height: 32px; + width: 100%; + font-size: 12px; + display: inline-block; + -webkit-box-sizing: border-box; + box-sizing: border-box; + border-radius: 16px; + padding-right: 10px; + padding-left: 30px; +} + +.el-transfer-panel__filter .el-input__icon { + margin-left: 5px; +} + +.el-transfer-panel .el-transfer-panel__header { + height: 40px; + line-height: 40px; + background: #f5f7fa; + margin: 0; + padding-left: 15px; + border-bottom: 1px solid #ebeef5; + -webkit-box-sizing: border-box; + box-sizing: border-box; + color: #000; +} + +.el-transfer-panel .el-transfer-panel__header .el-checkbox { + display: block; + line-height: 40px; +} + +.el-transfer-panel .el-transfer-panel__header .el-checkbox .el-checkbox__label { + font-size: 16px; + color: #303133; + font-weight: 400; +} + +.el-transfer-panel .el-transfer-panel__header .el-checkbox .el-checkbox__label span { + position: absolute; + right: 15px; + color: #909399; + font-size: 12px; + font-weight: 400; +} + +.el-divider__text, +.el-link { + font-weight: 500; + font-size: 14px; +} + +.el-transfer-panel .el-transfer-panel__footer { + height: 40px; + background: #fff; + margin: 0; + padding: 0; + border-top: 1px solid #ebeef5; + position: absolute; + bottom: 0; + left: 0; + width: 100%; + z-index: 1; +} + +.el-transfer-panel .el-transfer-panel__footer::after { + display: inline-block; + content: ''; + height: 100%; + vertical-align: middle; +} + +.el-container, +.el-timeline-item__node { + display: -webkit-box; + display: -ms-flexbox; +} + +.el-transfer-panel .el-transfer-panel__footer .el-checkbox { + padding-left: 20px; + color: #606266; +} + +.el-transfer-panel .el-transfer-panel__empty { + margin: 0; + height: 30px; + line-height: 30px; + padding: 6px 15px 0; + color: #909399; + text-align: center; +} + +.el-transfer-panel .el-checkbox__label { + padding-left: 8px; +} + +.el-transfer-panel .el-checkbox__inner { + height: 14px; + width: 14px; + border-radius: 3px; +} + +.el-transfer-panel .el-checkbox__inner::after { + height: 6px; + width: 3px; + left: 4px; +} + +.el-container { + display: flex; + -webkit-box-orient: horizontal; + -ms-flex-direction: row; + flex-direction: row; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + -ms-flex-preferred-size: auto; + flex-basis: auto; + -webkit-box-sizing: border-box; + box-sizing: border-box; + min-width: 0; +} + +.el-aside, +.el-header { + -webkit-box-sizing: border-box; +} + +.el-container.is-vertical { + -webkit-box-orient: vertical; + -ms-flex-direction: column; + flex-direction: column; +} + +.el-header { + padding: 0 20px; + box-sizing: border-box; + -ms-flex-negative: 0; + flex-shrink: 0; +} + +.el-aside { + overflow: auto; + box-sizing: border-box; + -ms-flex-negative: 0; + flex-shrink: 0; +} + +.el-footer, +.el-main { + -webkit-box-sizing: border-box; +} + +.el-main { + display: block; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + -ms-flex-preferred-size: auto; + flex-basis: auto; + overflow: auto; + box-sizing: border-box; + padding: 20px; +} + +.el-footer { + padding: 0 20px; + box-sizing: border-box; + -ms-flex-negative: 0; + flex-shrink: 0; +} + +.el-timeline { + margin: 0; + font-size: 14px; + list-style: none; +} + +.el-timeline .el-timeline-item:last-child .el-timeline-item__tail { + display: none; +} + +.el-timeline-item { + position: relative; + padding-bottom: 20px; +} + +.el-timeline-item__wrapper { + position: relative; + padding-left: 28px; + top: -3px; +} + +.el-timeline-item__tail { + position: absolute; + left: 4px; + height: 100%; + border-left: 2px solid #e4e7ed; +} + +.el-timeline-item__icon { + color: #fff; + font-size: 13px; +} + +.el-timeline-item__node { + position: absolute; + background-color: #e4e7ed; + border-radius: 50%; + display: flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-image__error, +.el-timeline-item__dot { + display: -webkit-box; + display: -ms-flexbox; +} + +.el-timeline-item__node--normal { + left: -1px; + width: 12px; + height: 12px; +} + +.el-timeline-item__node--large { + left: -2px; + width: 14px; + height: 14px; +} + +.el-timeline-item__node--primary { + background-color: #00a5a7; +} + +.el-timeline-item__node--success { + background-color: #67c23a; +} + +.el-timeline-item__node--warning { + background-color: #e6a23c; +} + +.el-timeline-item__node--danger { + background-color: #f56c6c; +} + +.el-timeline-item__node--info { + background-color: #909399; +} + +.el-timeline-item__dot { + position: absolute; + display: flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-timeline-item__content { + color: #303133; +} + +.el-timeline-item__timestamp { + color: #909399; + line-height: 1; + font-size: 13px; +} + +.el-timeline-item__timestamp.is-top { + margin-bottom: 8px; + padding-top: 4px; +} + +.el-timeline-item__timestamp.is-bottom { + margin-top: 8px; +} + +.el-link { + display: -webkit-inline-box; + display: -ms-inline-flexbox; + display: inline-flex; + -webkit-box-orient: horizontal; + -ms-flex-direction: row; + flex-direction: row; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + vertical-align: middle; + position: relative; + text-decoration: none; + outline: 0; + padding: 0; +} + +.el-link.is-underline:hover:after { + content: ''; + position: absolute; + left: 0; + right: 0; + height: 0; + bottom: 0; + border-bottom: 1px solid #00a5a7; +} + +.el-link.el-link--default:after, +.el-link.el-link--primary.is-underline:hover:after, +.el-link.el-link--primary:after { + border-color: #00a5a7; +} + +.el-link.is-disabled { + cursor: not-allowed; +} + +.el-link [class*='el-icon-'] + span { + margin-left: 5px; +} + +.el-link.el-link--default { + color: #606266; +} + +.el-link.el-link--default:hover { + color: #00a5a7; +} + +.el-link.el-link--default.is-disabled { + color: #c0c4cc; +} + +.el-link.el-link--primary { + color: #00a5a7; +} + +.el-link.el-link--primary:hover { + color: rgb(51, 183, 185); +} + +.el-link.el-link--primary.is-disabled { + color: rgb(128, 210, 211); +} + +.el-link.el-link--danger.is-underline:hover:after, +.el-link.el-link--danger:after { + border-color: #f56c6c; +} + +.el-link.el-link--danger { + color: #f56c6c; +} + +.el-link.el-link--danger:hover { + color: #f78989; +} + +.el-link.el-link--danger.is-disabled { + color: #fab6b6; +} + +.el-link.el-link--success.is-underline:hover:after, +.el-link.el-link--success:after { + border-color: #67c23a; +} + +.el-link.el-link--success { + color: #67c23a; +} + +.el-link.el-link--success:hover { + color: #85ce61; +} + +.el-link.el-link--success.is-disabled { + color: #b3e19d; +} + +.el-link.el-link--warning.is-underline:hover:after, +.el-link.el-link--warning:after { + border-color: #e6a23c; +} + +.el-link.el-link--warning { + color: #e6a23c; +} + +.el-link.el-link--warning:hover { + color: #ebb563; +} + +.el-link.el-link--warning.is-disabled { + color: #f3d19e; +} + +.el-link.el-link--info.is-underline:hover:after, +.el-link.el-link--info:after { + border-color: #909399; +} + +.el-link.el-link--info { + color: #909399; +} + +.el-link.el-link--info:hover { + color: #a6a9ad; +} + +.el-link.el-link--info.is-disabled { + color: #c8c9cc; +} + +.el-divider { + background-color: #dcdfe6; + position: relative; +} + +.el-divider--horizontal { + display: block; + height: 1px; + width: 100%; + margin: 24px 0; +} + +.el-divider--vertical { + display: inline-block; + width: 1px; + height: 1em; + margin: 0 8px; + vertical-align: middle; + position: relative; +} + +.el-divider__text { + position: absolute; + background-color: #fff; + padding: 0 20px; + color: #303133; +} + +.el-image__error, +.el-image__placeholder { + background: #f5f7fa; +} + +.el-divider__text.is-left { + left: 20px; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); +} + +.el-divider__text.is-center { + left: 50%; + -webkit-transform: translateX(-50%) translateY(-50%); + transform: translateX(-50%) translateY(-50%); +} + +.el-divider__text.is-right { + right: 20px; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); +} + +.el-image__error, +.el-image__inner, +.el-image__placeholder { + width: 100%; + height: 100%; +} + +.el-image { + position: relative; + display: inline-block; + overflow: hidden; +} + +.el-image__inner { + vertical-align: top; +} + +.el-image__inner--center { + position: relative; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + display: block; +} + +.el-image__error { + display: flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + font-size: 14px; + color: #c0c4cc; + vertical-align: middle; +} + +.el-image__preview { + cursor: pointer; +} + +.el-image-viewer__wrapper { + position: fixed; + top: 0; + right: 0; + bottom: 0; + left: 0; +} + +.el-image-viewer__btn { + position: absolute; + z-index: 1; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + border-radius: 50%; + opacity: 0.8; + cursor: pointer; + -webkit-box-sizing: border-box; + box-sizing: border-box; + user-select: none; +} + +.el-image-viewer__close { + top: 40px; + right: 40px; + width: 40px; + height: 40px; + font-size: 40px; +} + +.el-image-viewer__canvas { + width: 100%; + height: 100%; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; +} + +.el-image-viewer__actions { + left: 50%; + bottom: 30px; + -webkit-transform: translateX(-50%); + transform: translateX(-50%); + width: 282px; + height: 44px; + padding: 0 23px; + background-color: #606266; + border-color: #fff; + border-radius: 22px; +} + +.el-image-viewer__actions__inner { + width: 100%; + height: 100%; + text-align: justify; + cursor: default; + font-size: 23px; + color: #fff; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -ms-flex-pack: distribute; + justify-content: space-around; +} + +.el-image-viewer__next, +.el-image-viewer__prev { + top: 50%; + width: 44px; + height: 44px; + font-size: 24px; + color: #fff; + background-color: #606266; + border-color: #fff; +} + +.el-image-viewer__prev { + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + left: 40px; +} + +.el-image-viewer__next { + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + right: 40px; + text-indent: 2px; +} + +.el-image-viewer__mask { + position: absolute; + width: 100%; + height: 100%; + top: 0; + left: 0; + opacity: 0.5; + background: #000; +} + +.viewer-fade-enter-active { + -webkit-animation: viewer-fade-in 0.3s; + animation: viewer-fade-in 0.3s; +} + +.viewer-fade-leave-active { + -webkit-animation: viewer-fade-out 0.3s; + animation: viewer-fade-out 0.3s; +} + +@-webkit-keyframes viewer-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@keyframes viewer-fade-in { + 0% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } + + 100% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } +} + +@-webkit-keyframes viewer-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +@keyframes viewer-fade-out { + 0% { + -webkit-transform: translate3d(0, 0, 0); + transform: translate3d(0, 0, 0); + opacity: 1; + } + + 100% { + -webkit-transform: translate3d(0, -20px, 0); + transform: translate3d(0, -20px, 0); + opacity: 0; + } +} + +.el-calendar { + background-color: #fff; +} + +.el-calendar__header { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-pack: justify; + -ms-flex-pack: justify; + justify-content: space-between; + padding: 12px 20px; + border-bottom: 1px solid #ebeef5; +} + +.el-backtop, +.el-page-header { + display: -webkit-box; + display: -ms-flexbox; +} + +.el-calendar__title { + color: #000; + -ms-flex-item-align: center; + align-self: center; +} + +.el-calendar__body { + padding: 12px 20px 35px; +} + +.el-calendar-table { + table-layout: fixed; + width: 100%; +} + +.el-calendar-table thead th { + padding: 12px 0; + color: #606266; + font-weight: 400; +} + +.el-calendar-table:not(.is-range) td.next, +.el-calendar-table:not(.is-range) td.prev { + color: #c0c4cc; +} + +.el-backtop, +.el-calendar-table td.is-today { + color: #00a5a7; +} + +.el-calendar-table td { + border-bottom: 1px solid #ebeef5; + border-right: 1px solid #ebeef5; + vertical-align: top; + -webkit-transition: background-color 0.2s ease; + transition: background-color 0.2s ease; +} + +.el-calendar-table td.is-selected { + background-color: #f2f8fe; +} + +.el-calendar-table tr:first-child td { + border-top: 1px solid #ebeef5; +} + +.el-calendar-table tr td:first-child { + border-left: 1px solid #ebeef5; +} + +.el-calendar-table tr.el-calendar-table__row--hide-border td { + border-top: none; +} + +.el-calendar-table .el-calendar-day { + -webkit-box-sizing: border-box; + box-sizing: border-box; + padding: 8px; + height: 85px; +} + +.el-calendar-table .el-calendar-day:hover { + cursor: pointer; + background-color: #f2f8fe; +} + +.el-backtop { + position: fixed; + background-color: #fff; + width: 40px; + height: 40px; + border-radius: 50%; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + -webkit-box-pack: center; + -ms-flex-pack: center; + justify-content: center; + font-size: 20px; + -webkit-box-shadow: 0 0 6px rgba(0, 0, 0, 0.12); + box-shadow: 0 0 6px rgba(0, 0, 0, 0.12); + cursor: pointer; + z-index: 5; +} + +.el-backtop:hover { + background-color: #f2f6fc; +} + +.el-page-header { + display: flex; + line-height: 24px; +} + +.el-page-header__left { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + cursor: pointer; + margin-right: 40px; + position: relative; +} + +.el-page-header__left::after { + content: ''; + position: absolute; + width: 1px; + height: 16px; + right: -20px; + top: 50%; + -webkit-transform: translateY(-50%); + transform: translateY(-50%); + background-color: #dcdfe6; +} + +.el-checkbox, +.el-checkbox__input { + display: inline-block; + position: relative; + white-space: nowrap; +} + +.el-page-header__left .el-icon-back { + font-size: 18px; + margin-right: 6px; + -ms-flex-item-align: center; + align-self: center; +} + +.el-page-header__title { + font-size: 14px; + font-weight: 500; +} + +.el-page-header__content { + font-size: 18px; + color: #303133; +} + +.el-checkbox { + color: #606266; + font-weight: 500; + font-size: 14px; + cursor: pointer; + user-select: none; + margin-right: 30px; +} + +.el-checkbox-button__inner, +.el-radio { + font-weight: 500; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; +} + +.el-checkbox.is-bordered { + padding: 9px 20px 9px 10px; + border-radius: 4px; + border: 1px solid #dcdfe6; + -webkit-box-sizing: border-box; + box-sizing: border-box; + line-height: normal; + height: 40px; +} + +.el-checkbox.is-bordered.is-checked { + border-color: #00a5a7; +} + +.el-checkbox.is-bordered.is-disabled { + border-color: #ebeef5; + cursor: not-allowed; +} + +.el-checkbox.is-bordered + .el-checkbox.is-bordered { + margin-left: 10px; +} + +.el-checkbox.is-bordered.el-checkbox--medium { + padding: 7px 20px 7px 10px; + border-radius: 4px; + height: 36px; +} + +.el-checkbox.is-bordered.el-checkbox--medium .el-checkbox__label { + line-height: 17px; + font-size: 14px; +} + +.el-checkbox.is-bordered.el-checkbox--medium .el-checkbox__inner { + height: 14px; + width: 14px; +} + +.el-checkbox.is-bordered.el-checkbox--small { + padding: 5px 15px 5px 10px; + border-radius: 3px; + height: 32px; +} + +.el-checkbox.is-bordered.el-checkbox--small .el-checkbox__label { + line-height: 15px; + font-size: 12px; +} + +.el-checkbox.is-bordered.el-checkbox--small .el-checkbox__inner { + height: 12px; + width: 12px; +} + +.el-checkbox.is-bordered.el-checkbox--small .el-checkbox__inner::after { + height: 6px; + width: 2px; +} + +.el-checkbox.is-bordered.el-checkbox--mini { + padding: 3px 15px 3px 10px; + border-radius: 3px; + height: 28px; +} + +.el-checkbox.is-bordered.el-checkbox--mini .el-checkbox__label { + line-height: 12px; + font-size: 12px; +} + +.el-checkbox.is-bordered.el-checkbox--mini .el-checkbox__inner { + height: 12px; + width: 12px; +} + +.el-checkbox.is-bordered.el-checkbox--mini .el-checkbox__inner::after { + height: 6px; + width: 2px; +} + +.el-checkbox__input { + cursor: pointer; + outline: 0; + line-height: 1; + vertical-align: middle; +} + +.el-checkbox__input.is-disabled .el-checkbox__inner { + background-color: #edf2fc; + border-color: #dcdfe6; + cursor: not-allowed; +} + +.el-checkbox__input.is-disabled .el-checkbox__inner::after { + cursor: not-allowed; + border-color: #c0c4cc; +} + +.el-checkbox__input.is-disabled .el-checkbox__inner + .el-checkbox__label { + cursor: not-allowed; +} + +.el-checkbox__input.is-disabled.is-checked .el-checkbox__inner { + background-color: #f2f6fc; + border-color: #dcdfe6; +} + +.el-checkbox__input.is-disabled.is-checked .el-checkbox__inner::after { + border-color: #c0c4cc; +} + +.el-checkbox__input.is-disabled.is-indeterminate .el-checkbox__inner { + background-color: #f2f6fc; + border-color: #dcdfe6; +} + +.el-checkbox__input.is-disabled.is-indeterminate .el-checkbox__inner::before { + background-color: #c0c4cc; + border-color: #c0c4cc; +} + +.el-checkbox__input.is-checked .el-checkbox__inner, +.el-checkbox__input.is-indeterminate .el-checkbox__inner { + background-color: #00a5a7; + border-color: #00a5a7; +} + +.el-checkbox__input.is-disabled + span.el-checkbox__label { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-checkbox__input.is-checked .el-checkbox__inner::after { + -webkit-transform: rotate(45deg) scaleY(1); + transform: rotate(45deg) scaleY(1); +} + +.el-checkbox__input.is-checked + .el-checkbox__label { + color: #00a5a7; +} + +.el-checkbox__input.is-focus .el-checkbox__inner { + border-color: #00a5a7; +} + +.el-checkbox__input.is-indeterminate .el-checkbox__inner::before { + content: ''; + position: absolute; + display: block; + background-color: #fff; + height: 2px; + -webkit-transform: scale(0.5); + transform: scale(0.5); + left: 0; + right: 0; + top: 5px; +} + +.el-checkbox__input.is-indeterminate .el-checkbox__inner::after { + display: none; +} + +.el-checkbox__inner { + display: inline-block; + position: relative; + border: 1px solid #b8becc; + border-radius: 2px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + width: 14px; + height: 14px; + background-color: #fff; + z-index: 1; + -webkit-transition: border-color 0.25s cubic-bezier(0.71, -0.46, 0.29, 1.46), + background-color 0.25s cubic-bezier(0.71, -0.46, 0.29, 1.46); + transition: border-color 0.25s cubic-bezier(0.71, -0.46, 0.29, 1.46), + background-color 0.25s cubic-bezier(0.71, -0.46, 0.29, 1.46); +} + +.el-checkbox__inner:hover { + border-color: #00a5a7; +} + +.el-checkbox__inner::after { + -webkit-box-sizing: content-box; + box-sizing: content-box; + content: ''; + border: 1px solid #fff; + border-left: 0; + border-top: 0; + height: 7px; + left: 4px; + position: absolute; + top: 1px; + -webkit-transform: rotate(45deg) scaleY(0); + transform: rotate(45deg) scaleY(0); + width: 3px; + -webkit-transition: -webkit-transform 0.15s ease-in 0.05s; + transition: -webkit-transform 0.15s ease-in 0.05s; + transition: transform 0.15s ease-in 0.05s; + transition: transform 0.15s ease-in 0.05s, -webkit-transform 0.15s ease-in 0.05s; + -webkit-transform-origin: center; + transform-origin: center; +} + +.el-checkbox__original { + opacity: 0; + outline: 0; + position: absolute; + margin: 0; + width: 0; + height: 0; + z-index: -1; +} + +.el-checkbox-button, +.el-checkbox-button__inner { + display: inline-block; + position: relative; +} + +.el-checkbox__label { + display: inline-block; + padding-left: 10px; + line-height: 19px; + font-size: 14px; +} + +.el-checkbox:last-of-type { + margin-right: 0; +} + +.el-checkbox-button__inner { + line-height: 1; + white-space: nowrap; + vertical-align: middle; + cursor: pointer; + background: #fff; + border: 1px solid #dcdfe6; + border-left: 0; + color: #606266; + -webkit-appearance: none; + text-align: center; + -webkit-box-sizing: border-box; + box-sizing: border-box; + outline: 0; + margin: 0; + -webkit-transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + transition: all 0.3s cubic-bezier(0.645, 0.045, 0.355, 1); + padding: 12px 20px; + font-size: 14px; + border-radius: 0; +} + +.el-checkbox-button__inner.is-round { + padding: 12px 20px; +} + +.el-checkbox-button__inner:hover { + color: #00a5a7; +} + +.el-checkbox-button__inner [class*='el-icon-'] { + line-height: 0.9; +} + +.el-radio, +.el-radio__input { + line-height: 1; + outline: 0; + white-space: nowrap; +} + +.el-checkbox-button__inner [class*='el-icon-'] + span { + margin-left: 5px; +} + +.el-checkbox-button__original { + opacity: 0; + outline: 0; + position: absolute; + margin: 0; + z-index: -1; +} + +.el-radio, +.el-radio__inner, +.el-radio__input { + position: relative; + display: inline-block; +} + +.el-checkbox-button.is-checked .el-checkbox-button__inner { + color: #fff; + background-color: #00a5a7; + border-color: #00a5a7; + -webkit-box-shadow: -1px 0 0 0 rgb(102, 201, 202); + box-shadow: -1px 0 0 0 rgb(102, 201, 202); +} + +.el-checkbox-button.is-checked:first-child .el-checkbox-button__inner { + border-left-color: #00a5a7; +} + +.el-checkbox-button.is-disabled .el-checkbox-button__inner { + color: #c0c4cc; + cursor: not-allowed; + background-image: none; + background-color: #fff; + border-color: #ebeef5; + -webkit-box-shadow: none; + box-shadow: none; +} + +.el-checkbox-button.is-disabled:first-child .el-checkbox-button__inner { + border-left-color: #ebeef5; +} + +.el-checkbox-button:first-child .el-checkbox-button__inner { + border-left: 1px solid #dcdfe6; + border-radius: 4px 0 0 4px; + -webkit-box-shadow: none !important; + box-shadow: none !important; +} + +.el-checkbox-button.is-focus .el-checkbox-button__inner { + border-color: #00a5a7; +} + +.el-checkbox-button:last-child .el-checkbox-button__inner { + border-radius: 0 4px 4px 0; +} + +.el-checkbox-button--medium .el-checkbox-button__inner { + padding: 10px 20px; + font-size: 14px; + border-radius: 0; +} + +.el-checkbox-button--medium .el-checkbox-button__inner.is-round { + padding: 10px 20px; +} + +.el-checkbox-button--small .el-checkbox-button__inner { + padding: 9px 15px; + font-size: 12px; + border-radius: 0; +} + +.el-checkbox-button--small .el-checkbox-button__inner.is-round { + padding: 9px 15px; +} + +.el-checkbox-button--mini .el-checkbox-button__inner { + padding: 7px 15px; + font-size: 12px; + border-radius: 0; +} + +.el-checkbox-button--mini .el-checkbox-button__inner.is-round { + padding: 7px 15px; +} + +.el-checkbox-group { + font-size: 0; +} + +.el-radio, +.el-radio--medium.is-bordered .el-radio__label { + font-size: 14px; +} + +.el-radio { + color: #606266; + cursor: pointer; + margin-right: 30px; +} + +.el-cascader-node > .el-radio, +.el-radio:last-child { + margin-right: 0; +} + +.el-radio.is-bordered { + padding: 12px 20px 0 10px; + border-radius: 4px; + border: 1px solid #dcdfe6; + -webkit-box-sizing: border-box; + box-sizing: border-box; + height: 40px; +} + +.el-radio.is-bordered.is-checked { + border-color: #00a5a7; +} + +.el-radio.is-bordered.is-disabled { + cursor: not-allowed; + border-color: #ebeef5; +} + +.el-radio__input.is-disabled .el-radio__inner, +.el-radio__input.is-disabled.is-checked .el-radio__inner { + background-color: #f5f7fa; + border-color: #e4e7ed; +} + +.el-radio.is-bordered + .el-radio.is-bordered { + margin-left: 10px; +} + +.el-radio--medium.is-bordered { + padding: 10px 20px 0 10px; + border-radius: 4px; + height: 36px; +} + +.el-radio--mini.is-bordered .el-radio__label, +.el-radio--small.is-bordered .el-radio__label { + font-size: 12px; +} + +.el-radio--medium.is-bordered .el-radio__inner { + height: 14px; + width: 14px; +} + +.el-radio--small.is-bordered { + padding: 8px 15px 0 10px; + border-radius: 3px; + height: 32px; +} + +.el-radio--small.is-bordered .el-radio__inner { + height: 12px; + width: 12px; +} + +.el-radio--mini.is-bordered { + padding: 6px 15px 0 10px; + border-radius: 3px; + height: 28px; +} + +.el-radio--mini.is-bordered .el-radio__inner { + height: 12px; + width: 12px; +} + +.el-radio__input { + cursor: pointer; + vertical-align: middle; +} + +.el-radio__input.is-disabled .el-radio__inner { + cursor: not-allowed; +} + +.el-radio__input.is-disabled .el-radio__inner::after { + cursor: not-allowed; + background-color: #f5f7fa; +} + +.el-radio__input.is-disabled .el-radio__inner + .el-radio__label { + cursor: not-allowed; +} + +.el-radio__input.is-disabled.is-checked .el-radio__inner::after { + background-color: #c0c4cc; +} + +.el-radio__input.is-disabled + span.el-radio__label { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-radio__input.is-checked .el-radio__inner { + border-color: #00a5a7; + background: #00a5a7; +} + +.el-radio__input.is-checked .el-radio__inner::after { + -webkit-transform: translate(-50%, -50%) scale(1); + transform: translate(-50%, -50%) scale(1); +} + +.el-radio__input.is-checked + .el-radio__label { + color: #00a5a7; +} + +.el-radio__input.is-focus .el-radio__inner { + border-color: #00a5a7; +} + +.el-radio__inner { + border: 1px solid #b8becc; + border-radius: 100%; + width: 14px; + height: 14px; + background-color: #fff; + cursor: pointer; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-radio__inner:hover { + border-color: #00a5a7; +} + +.el-radio__inner::after { + width: 4px; + height: 4px; + border-radius: 100%; + background-color: #fff; + content: ''; + position: absolute; + left: 50%; + top: 50%; + -webkit-transform: translate(-50%, -50%) scale(0); + transform: translate(-50%, -50%) scale(0); + -webkit-transition: -webkit-transform 0.15s ease-in; + transition: -webkit-transform 0.15s ease-in; + transition: transform 0.15s ease-in; + transition: transform 0.15s ease-in, -webkit-transform 0.15s ease-in; +} + +.el-radio__original { + opacity: 0; + outline: 0; + position: absolute; + z-index: -1; + top: 0; + left: 0; + right: 0; + bottom: 0; + margin: 0; +} + +.el-radio:focus:not(.is-focus):not(:active):not(.is-disabled) .el-radio__inner { + -webkit-box-shadow: 0 0 2px 2px #00a5a7; + box-shadow: 0 0 2px 2px #00a5a7; +} + +.el-radio__label { + font-size: 14px; + padding-left: 10px; +} + +.el-scrollbar { + overflow: hidden; + position: relative; +} + +.el-scrollbar:active > .el-scrollbar__bar, +.el-scrollbar:focus > .el-scrollbar__bar, +.el-scrollbar:hover > .el-scrollbar__bar { + opacity: 1; + -webkit-transition: opacity 340ms ease-out; + transition: opacity 340ms ease-out; +} + +.el-scrollbar__wrap { + overflow: scroll; + height: 100%; +} + +.el-scrollbar__wrap--hidden-default::-webkit-scrollbar { + width: 0; + height: 0; +} + +.el-scrollbar__thumb { + position: relative; + display: block; + width: 0; + height: 0; + cursor: pointer; + border-radius: inherit; + background-color: rgba(144, 147, 153, 0.3); + -webkit-transition: 0.3s background-color; + transition: 0.3s background-color; +} + +.el-scrollbar__thumb:hover { + background-color: rgba(144, 147, 153, 0.5); +} + +.el-scrollbar__bar { + position: absolute; + right: 2px; + bottom: 2px; + z-index: 1; + border-radius: 4px; + opacity: 0; + -webkit-transition: opacity 120ms ease-out; + transition: opacity 120ms ease-out; +} + +.el-scrollbar__bar.is-vertical { + width: 6px; + top: 2px; +} + +.el-scrollbar__bar.is-vertical > div { + width: 100%; +} + +.el-scrollbar__bar.is-horizontal { + height: 6px; + left: 2px; +} + +.el-scrollbar__bar.is-horizontal > div { + height: 100%; +} + +.el-cascader-panel { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + border-radius: 4px; + font-size: 14px; +} + +.el-cascader-node, +.el-drawer { + display: -webkit-box; + display: -ms-flexbox; +} + +.el-cascader-panel.is-bordered { + border: 1px solid #e4e7ed; + border-radius: 4px; +} + +.el-cascader-menu { + min-width: 180px; + -webkit-box-sizing: border-box; + box-sizing: border-box; + color: #606266; + border-right: solid 1px #e4e7ed; +} + +.el-cascader-menu:last-child { + border-right: none; +} + +.el-cascader-menu:last-child .el-cascader-node { + padding-right: 20px; +} + +.el-cascader-menu__wrap { + height: 204px; +} + +.el-cascader-menu__list { + position: relative; + min-height: 100%; + margin: 0; + padding: 6px 0; + list-style: none; + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-avatar, +.el-drawer { + -webkit-box-sizing: border-box; + overflow: hidden; +} + +.el-cascader-menu__hover-zone { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + pointer-events: none; +} + +.el-cascader-menu__empty-text { + position: absolute; + top: 50%; + left: 50%; + -webkit-transform: translate(-50%, -50%); + transform: translate(-50%, -50%); + text-align: center; + color: #c0c4cc; +} + +.el-cascader-node { + position: relative; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + padding: 0 30px 0 20px; + height: 34px; + line-height: 34px; + outline: 0; +} + +.el-cascader-node.is-selectable.in-active-path { + color: #606266; +} + +.el-cascader-node.in-active-path, +.el-cascader-node.is-active, +.el-cascader-node.is-selectable.in-checked-path { + color: #00a5a7; + font-weight: 700; +} + +.el-cascader-node:not(.is-disabled) { + cursor: pointer; +} + +.el-cascader-node:not(.is-disabled):focus, +.el-cascader-node:not(.is-disabled):hover { + background: #f5f7fa; +} + +.el-cascader-node.is-disabled { + color: #c0c4cc; + cursor: not-allowed; +} + +.el-cascader-node__prefix { + position: absolute; + left: 10px; +} + +.el-cascader-node__postfix { + position: absolute; + right: 10px; +} + +.el-cascader-node__label { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + padding: 0 10px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.el-cascader-node > .el-radio .el-radio__label { + padding-left: 0; +} + +.el-avatar { + display: inline-block; + box-sizing: border-box; + text-align: center; + color: #fff; + background: #c0c4cc; + width: 40px; + height: 40px; + line-height: 40px; + font-size: 14px; +} + +.el-avatar > img { + display: block; + height: 100%; + vertical-align: middle; +} + +.el-avatar--circle { + border-radius: 50%; +} + +.el-avatar--square { + border-radius: 4px; +} + +.el-avatar--icon { + font-size: 18px; +} + +.el-avatar--large { + width: 40px; + height: 40px; + line-height: 40px; +} + +.el-avatar--medium { + width: 36px; + height: 36px; + line-height: 36px; +} + +.el-avatar--small { + width: 28px; + height: 28px; + line-height: 28px; +} + +.el-drawer.btt, +.el-drawer.ttb, +.el-drawer__container { + left: 0; + right: 0; + width: 100%; +} + +.el-drawer.ltr, +.el-drawer.rtl, +.el-drawer__container { + top: 0; + bottom: 0; + height: 100%; +} + +@-webkit-keyframes el-drawer-fade-in { + 0% { + opacity: 0; + } + + 100% { + opacity: 1; + } +} + +@keyframes el-drawer-fade-in { + 0% { + opacity: 0; + } + + 100% { + opacity: 1; + } +} + +@-webkit-keyframes rtl-drawer-in { + 0% { + -webkit-transform: translate(100%, 0); + transform: translate(100%, 0); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@keyframes rtl-drawer-in { + 0% { + -webkit-transform: translate(100%, 0); + transform: translate(100%, 0); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@-webkit-keyframes rtl-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(100%, 0); + transform: translate(100%, 0); + } +} + +@keyframes rtl-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(100%, 0); + transform: translate(100%, 0); + } +} + +@-webkit-keyframes ltr-drawer-in { + 0% { + -webkit-transform: translate(-100%, 0); + transform: translate(-100%, 0); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@keyframes ltr-drawer-in { + 0% { + -webkit-transform: translate(-100%, 0); + transform: translate(-100%, 0); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@-webkit-keyframes ltr-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(-100%, 0); + transform: translate(-100%, 0); + } +} + +@keyframes ltr-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(-100%, 0); + transform: translate(-100%, 0); + } +} + +@-webkit-keyframes ttb-drawer-in { + 0% { + -webkit-transform: translate(0, -100%); + transform: translate(0, -100%); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@keyframes ttb-drawer-in { + 0% { + -webkit-transform: translate(0, -100%); + transform: translate(0, -100%); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@-webkit-keyframes ttb-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(0, -100%); + transform: translate(0, -100%); + } +} + +@keyframes ttb-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(0, -100%); + transform: translate(0, -100%); + } +} + +@-webkit-keyframes btt-drawer-in { + 0% { + -webkit-transform: translate(0, 100%); + transform: translate(0, 100%); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@keyframes btt-drawer-in { + 0% { + -webkit-transform: translate(0, 100%); + transform: translate(0, 100%); + } + + 100% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } +} + +@-webkit-keyframes btt-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(0, 100%); + transform: translate(0, 100%); + } +} + +@keyframes btt-drawer-out { + 0% { + -webkit-transform: translate(0, 0); + transform: translate(0, 0); + } + + 100% { + -webkit-transform: translate(0, 100%); + transform: translate(0, 100%); + } +} + +.el-drawer { + position: absolute; + box-sizing: border-box; + background-color: #fff; + display: flex; + -webkit-box-orient: vertical; + -ms-flex-direction: column; + flex-direction: column; + -webkit-box-shadow: 0 8px 10px -5px rgba(0, 0, 0, 0.2), 0 16px 24px 2px rgba(0, 0, 0, 0.14), + 0 6px 30px 5px rgba(0, 0, 0, 0.12); + box-shadow: 0 8px 10px -5px rgba(0, 0, 0, 0.2), 0 16px 24px 2px rgba(0, 0, 0, 0.14), + 0 6px 30px 5px rgba(0, 0, 0, 0.12); +} + +.el-drawer.rtl { + -webkit-animation: rtl-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: rtl-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + right: 0; +} + +.el-drawer__open .el-drawer.rtl { + -webkit-animation: rtl-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: rtl-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; +} + +.el-drawer.ltr { + -webkit-animation: ltr-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: ltr-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + left: 0; +} + +.el-drawer__open .el-drawer.ltr { + -webkit-animation: ltr-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: ltr-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; +} + +.el-drawer.ttb { + -webkit-animation: ttb-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: ttb-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + top: 0; +} + +.el-drawer__open .el-drawer.ttb { + -webkit-animation: ttb-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: ttb-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; +} + +.el-drawer.btt { + -webkit-animation: btt-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: btt-drawer-out 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + bottom: 0; +} + +.el-drawer__open .el-drawer.btt { + -webkit-animation: btt-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: btt-drawer-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; +} + +.el-drawer__header { + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + color: #72767b; + display: -webkit-box; + display: -ms-flexbox; + display: flex; + margin-bottom: 32px; + padding: 20px 20px 0; +} + +.el-drawer__header > :first-child { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; +} + +.el-drawer__title { + margin: 0; + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; + line-height: inherit; + font-size: 1rem; +} + +.el-drawer__close-btn { + border: none; + cursor: pointer; + font-size: 20px; + color: inherit; + background-color: transparent; +} + +.el-drawer__body { + -webkit-box-flex: 1; + -ms-flex: 1; + flex: 1; +} + +.el-drawer__body > * { + -webkit-box-sizing: border-box; + box-sizing: border-box; +} + +.el-drawer__container { + position: relative; +} + +.el-drawer-fade-enter-active { + -webkit-animation: el-drawer-fade-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; + animation: el-drawer-fade-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s; +} + +.el-drawer-fade-leave-active { + animation: el-drawer-fade-in 225ms cubic-bezier(0, 0, 0.2, 1) 0s reverse; +} diff --git a/mindinsight/ui/src/assets/css/reset.scss b/mindinsight/ui/src/assets/css/reset.scss new file mode 100644 index 00000000..3246f079 --- /dev/null +++ b/mindinsight/ui/src/assets/css/reset.scss @@ -0,0 +1,182 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import './variable.scss'; + +html, +body, +div, +span, +applet, +object, +iframe, +h1, +h2, +h3, +h4, +h5, +h6, +p, +blockquote, +pre, +a, +abbr, +acronym, +address, +big, +cite, +code, +del, +dfn, +em, +font, +img, +ins, +kbd, +q, +s, +samp, +small, +strike, +strong, +sub, +sup, +tt, +var, +dl, +dt, +dd, +ol, +ul, +li, +fieldset, +form, +label, +legend, +table, +caption, +tbody, +tfoot, +thead, +tr, +th, +td { + margin: 0; + padding: 0; + border: 0; + outline: 0; + font-weight: inherit; + font-style: inherit; + font-family: $bodyFontFamily; + font-size: $bodyFontSize; + vertical-align: baseline; + box-sizing: border-box; +} +:focus { + outline: 0; +} + +a, +a:link, +a:visited, +a:hover, +a:active { + text-decoration: none; +} + +a { + color: $linkColor; +} +a:hover { + color: $linkActiveColor; +} +table { + border-collapse: separate; + border-spacing: 0; +} +th, +td { + text-align: left; + font-weight: normal; +} +img, +iframe { + border: none; + text-decoration: none; +} +ol, +ul { + list-style: none; +} +input, +textarea, +select, +button { + font-size: 100%; + font-family: inherit; +} +select { + margin: inherit; +} +hr { + margin: 0; + padding: 0; + border: 0; + color: $bodyColor; + background-color: $bodyColor; + height: 1px; +} + +html, +body { + height: 100%; + background-color: $bodyBg; + font-family: $bodyFontFamily; + font-size: $bodyFontSize; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + color: $bodyColor; + box-sizing: border-box; +} + +/* + * Defines the width of the scroll bar and the width of the background, respectively corresponding to the dimensions of the horizontal and vertical scroll bars. + */ +::-webkit-scrollbar { + width: 6px; + height: 6px; +} + +/* + * Defines the shadow + rounded corners of the scroll bar. + */ +::-webkit-scrollbar-track { + background-color: #f5f5f5; +} + +/* + * Defines the shadow + in the slider. + */ +::-webkit-scrollbar-thumb { + background-color: #dfe7f5; +} +.select-disable { + -moz-user-select: none; /* Firefox */ + -webkit-user-select: none; /* webkitbrowser */ + -ms-user-select: none; /* IE10 */ + -khtml-user-select: none; /* Early browser */ + user-select: none; +} diff --git a/mindinsight/ui/src/assets/css/variable.scss b/mindinsight/ui/src/assets/css/variable.scss new file mode 100644 index 00000000..d503342d --- /dev/null +++ b/mindinsight/ui/src/assets/css/variable.scss @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* body */ +$bodyBg: #fff; // Default background color +$bodyColor: #333; // Default font color +$bodyFontSize: 14px; // Default font size +$bodyFontFamily: 'Helvetica Neue', Helvetica, 'PingFang SC', 'Hiragino Sans GB', + 'Microsoft YaHei', Arial, sans-serif; // Default font + +/* link */ +$linkColor: #3399ff; +$linkActiveColor: #3399ff; + +/* header */ +$headerBackground: linear-gradient( + 180deg, + #263d5f 0%, + #16233b 100% +); // Head background color +$headerHeight: 64px; // Head height +$headerColor: #fff; // Head font color diff --git a/mindinsight/ui/src/assets/fonts/element-icons.ttf b/mindinsight/ui/src/assets/fonts/element-icons.ttf new file mode 100644 index 00000000..91b74de3 Binary files /dev/null and b/mindinsight/ui/src/assets/fonts/element-icons.ttf differ diff --git a/mindinsight/ui/src/assets/fonts/element-icons.woff b/mindinsight/ui/src/assets/fonts/element-icons.woff new file mode 100644 index 00000000..02b9a253 Binary files /dev/null and b/mindinsight/ui/src/assets/fonts/element-icons.woff differ diff --git a/mindinsight/ui/src/assets/images/all-drop-down.png b/mindinsight/ui/src/assets/images/all-drop-down.png new file mode 100644 index 00000000..02feded0 Binary files /dev/null and b/mindinsight/ui/src/assets/images/all-drop-down.png differ diff --git a/mindinsight/ui/src/assets/images/all-uptake.png b/mindinsight/ui/src/assets/images/all-uptake.png new file mode 100644 index 00000000..5bc3f300 Binary files /dev/null and b/mindinsight/ui/src/assets/images/all-uptake.png differ diff --git a/mindinsight/ui/src/assets/images/cancel-warm-text.png b/mindinsight/ui/src/assets/images/cancel-warm-text.png new file mode 100644 index 00000000..e7e55e93 Binary files /dev/null and b/mindinsight/ui/src/assets/images/cancel-warm-text.png differ diff --git a/mindinsight/ui/src/assets/images/close-page.png b/mindinsight/ui/src/assets/images/close-page.png new file mode 100644 index 00000000..c1540c48 Binary files /dev/null and b/mindinsight/ui/src/assets/images/close-page.png differ diff --git a/mindinsight/ui/src/assets/images/const.png b/mindinsight/ui/src/assets/images/const.png new file mode 100644 index 00000000..52dc30ea Binary files /dev/null and b/mindinsight/ui/src/assets/images/const.png differ diff --git a/mindinsight/ui/src/assets/images/constant-node.png b/mindinsight/ui/src/assets/images/constant-node.png new file mode 100644 index 00000000..2162211f Binary files /dev/null and b/mindinsight/ui/src/assets/images/constant-node.png differ diff --git a/mindinsight/ui/src/assets/images/control-dep.png b/mindinsight/ui/src/assets/images/control-dep.png new file mode 100644 index 00000000..ad716049 Binary files /dev/null and b/mindinsight/ui/src/assets/images/control-dep.png differ diff --git a/mindinsight/ui/src/assets/images/creat-dataset.png b/mindinsight/ui/src/assets/images/creat-dataset.png new file mode 100644 index 00000000..5f848a3e Binary files /dev/null and b/mindinsight/ui/src/assets/images/creat-dataset.png differ diff --git a/mindinsight/ui/src/assets/images/data-flow.png b/mindinsight/ui/src/assets/images/data-flow.png new file mode 100644 index 00000000..9c9b3ee2 Binary files /dev/null and b/mindinsight/ui/src/assets/images/data-flow.png differ diff --git a/mindinsight/ui/src/assets/images/download.png b/mindinsight/ui/src/assets/images/download.png new file mode 100644 index 00000000..6782f1f9 Binary files /dev/null and b/mindinsight/ui/src/assets/images/download.png differ diff --git a/mindinsight/ui/src/assets/images/full-screen.png b/mindinsight/ui/src/assets/images/full-screen.png new file mode 100644 index 00000000..36553c31 Binary files /dev/null and b/mindinsight/ui/src/assets/images/full-screen.png differ diff --git a/mindinsight/ui/src/assets/images/logo.png b/mindinsight/ui/src/assets/images/logo.png new file mode 100644 index 00000000..57c8762f Binary files /dev/null and b/mindinsight/ui/src/assets/images/logo.png differ diff --git a/mindinsight/ui/src/assets/images/map-dataset.png b/mindinsight/ui/src/assets/images/map-dataset.png new file mode 100644 index 00000000..5dc28c25 Binary files /dev/null and b/mindinsight/ui/src/assets/images/map-dataset.png differ diff --git a/mindinsight/ui/src/assets/images/mult-select.png b/mindinsight/ui/src/assets/images/mult-select.png new file mode 100644 index 00000000..78d4c4f8 Binary files /dev/null and b/mindinsight/ui/src/assets/images/mult-select.png differ diff --git a/mindinsight/ui/src/assets/images/mult-unselect.png b/mindinsight/ui/src/assets/images/mult-unselect.png new file mode 100644 index 00000000..644ca79f Binary files /dev/null and b/mindinsight/ui/src/assets/images/mult-unselect.png differ diff --git a/mindinsight/ui/src/assets/images/name-scope.png b/mindinsight/ui/src/assets/images/name-scope.png new file mode 100644 index 00000000..4e33e7a0 Binary files /dev/null and b/mindinsight/ui/src/assets/images/name-scope.png differ diff --git a/mindinsight/ui/src/assets/images/nodata.png b/mindinsight/ui/src/assets/images/nodata.png new file mode 100644 index 00000000..7dc49585 Binary files /dev/null and b/mindinsight/ui/src/assets/images/nodata.png differ diff --git a/mindinsight/ui/src/assets/images/operator-node.png b/mindinsight/ui/src/assets/images/operator-node.png new file mode 100644 index 00000000..0e8243ad Binary files /dev/null and b/mindinsight/ui/src/assets/images/operator-node.png differ diff --git a/mindinsight/ui/src/assets/images/polymetric.png b/mindinsight/ui/src/assets/images/polymetric.png new file mode 100644 index 00000000..0aa422ec Binary files /dev/null and b/mindinsight/ui/src/assets/images/polymetric.png differ diff --git a/mindinsight/ui/src/assets/images/reload.png b/mindinsight/ui/src/assets/images/reload.png new file mode 100644 index 00000000..0e867522 Binary files /dev/null and b/mindinsight/ui/src/assets/images/reload.png differ diff --git a/mindinsight/ui/src/assets/images/repeat-dataset.png b/mindinsight/ui/src/assets/images/repeat-dataset.png new file mode 100644 index 00000000..ceed208c Binary files /dev/null and b/mindinsight/ui/src/assets/images/repeat-dataset.png differ diff --git a/mindinsight/ui/src/assets/images/scroll-btn-down.png b/mindinsight/ui/src/assets/images/scroll-btn-down.png new file mode 100644 index 00000000..5d86e9a7 Binary files /dev/null and b/mindinsight/ui/src/assets/images/scroll-btn-down.png differ diff --git a/mindinsight/ui/src/assets/images/scroll-btn-left.png b/mindinsight/ui/src/assets/images/scroll-btn-left.png new file mode 100644 index 00000000..4f442a88 Binary files /dev/null and b/mindinsight/ui/src/assets/images/scroll-btn-left.png differ diff --git a/mindinsight/ui/src/assets/images/scroll-btn-right.png b/mindinsight/ui/src/assets/images/scroll-btn-right.png new file mode 100644 index 00000000..99d9a65a Binary files /dev/null and b/mindinsight/ui/src/assets/images/scroll-btn-right.png differ diff --git a/mindinsight/ui/src/assets/images/scroll-btn-up.png b/mindinsight/ui/src/assets/images/scroll-btn-up.png new file mode 100644 index 00000000..80d6980f Binary files /dev/null and b/mindinsight/ui/src/assets/images/scroll-btn-up.png differ diff --git a/mindinsight/ui/src/assets/images/shuffle-dataset.png b/mindinsight/ui/src/assets/images/shuffle-dataset.png new file mode 100644 index 00000000..49141bd6 Binary files /dev/null and b/mindinsight/ui/src/assets/images/shuffle-dataset.png differ diff --git a/mindinsight/ui/src/assets/images/toggle-right-bg.png b/mindinsight/ui/src/assets/images/toggle-right-bg.png new file mode 100644 index 00000000..b72e0796 Binary files /dev/null and b/mindinsight/ui/src/assets/images/toggle-right-bg.png differ diff --git a/mindinsight/ui/src/assets/images/toggle-right-icon.png b/mindinsight/ui/src/assets/images/toggle-right-icon.png new file mode 100644 index 00000000..ba008a47 Binary files /dev/null and b/mindinsight/ui/src/assets/images/toggle-right-icon.png differ diff --git a/mindinsight/ui/src/assets/images/virtual-node.png b/mindinsight/ui/src/assets/images/virtual-node.png new file mode 100644 index 00000000..8266ce6e Binary files /dev/null and b/mindinsight/ui/src/assets/images/virtual-node.png differ diff --git a/mindinsight/ui/src/common/common-property.js b/mindinsight/ui/src/common/common-property.js new file mode 100644 index 00000000..dbd5e944 --- /dev/null +++ b/mindinsight/ui/src/common/common-property.js @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export default { + commonColorArr: [ + '#6C92FA', + '#FA8E5A', + '#A6DD82', + '#6CBFFF', + '#F45C5E', + '#F6DF66', + '#4EDED2', + '#F3689A', + '#FDCA5A', + '#7ADFA0', + '#4C6BC2', + '#C2663A', + '#7EB05D', + '#468BC2', + '#BF5254', + '#C2AE44', + '#33B0A6', + '#BA456F', + '#C49939', + '#56B077', + '#A1BAFF', + '#FFBA99', + '#C7EDAD', + '#9ED5FF', + '#FA9698', + '#FAEB9D', + '#8CEDE5', + '#F79CBC', + '#FFDE96', + '#A8EDC2', + '#25386E', + '#8A4321', + '#59823D', + '#285C85', + '#873233', + '#8F7E29', + '#1E827A', + '#822849', + '#8C6A20', + '#388252', + '#DCFCE8', + '#FFE4D6', + '#D2FCF9', + '#FFF2D4', + '#D1EBFF', + '#FFF9D9', + '#D4DFFF', + '#FCD2E1', + '#E9FCDC', + '#FFD1D2', + ], + // define graph color array + graphColorArr: ['#FFF8F5', '#FDEEE7', '#FFE5D9', '#FFDAC9'], + // define fullscreen icon + fullScreenIcon: + 'path://M432.45,595.444c0,2.177-4.661,6.82-11.305,6.82c-6.475,' + + '0-11.306-4.567-11.306-6.82s4.852-6.812,11.306-6.812C427.841,' + + '588.632,432.452,593.191,432.45,595.444L432.45,595.444z M421.155,' + + '589.876c-3.009,0-5.448,2.495-5.448,5.572s2.439,5.572,5.448,' + + '5.572c3.01,0,5.449-2.495,5.449-5.572C426.604,592.371,424.165,' + + '589.876,421.155,589.876L421.155,589.876z M421.146,591.891c-1.916,' + + '0-3.47,1.589-3.47,3.549c0,1.959,1.554,3.548,3.47,3.548s3.469-1.589,' + + '3.469-3.548C424.614,593.479,423.062,591.891,421.146,591.891L421.146,591.891zM421.146,591.891', + // define svg style for graph download + graphDownloadStyle: + '', +}; diff --git a/mindinsight/ui/src/components/autocomplete.vue b/mindinsight/ui/src/components/autocomplete.vue new file mode 100644 index 00000000..81419d78 --- /dev/null +++ b/mindinsight/ui/src/components/autocomplete.vue @@ -0,0 +1,93 @@ + + + + diff --git a/mindinsight/ui/src/components/header.vue b/mindinsight/ui/src/components/header.vue new file mode 100644 index 00000000..5365f7dc --- /dev/null +++ b/mindinsight/ui/src/components/header.vue @@ -0,0 +1,261 @@ + + + + + diff --git a/mindinsight/ui/src/i18n.js b/mindinsight/ui/src/i18n.js new file mode 100644 index 00000000..b8f4d460 --- /dev/null +++ b/mindinsight/ui/src/i18n.js @@ -0,0 +1,55 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Vue from 'vue'; +import VueI18n from 'vue-i18n'; + +Vue.use(VueI18n); + +/** + * return language package + * @return {object} + */ +function loadLocaleMessages() { + const locales = require.context('./locales', true, /[A-Za-z0-9-_,\s]+\.json$/i); + + const messages = {}; + locales.keys().forEach((key) => { + const matched = key.match(/([A-Za-z0-9-_]+)\./i); + if (matched && matched.length > 1) { + const locale = matched[1]; + messages[locale] = locales(key); + } + }); + return messages; +} + +/* load default language pack */ +const langStorge = window.localStorage.getItem('milang'); +let langflag; +// Check language by default +if (langStorge) { + langflag = langStorge; +} else { + // set Chinese if no default language + langflag = 'zh-cn'; + window.localStorage.setItem('milang', langflag); +} + +export default new VueI18n({ + locale: langflag, + fallbackLocale: langflag, + messages: loadLocaleMessages(), +}); diff --git a/mindinsight/ui/src/locales/zh-cn.json b/mindinsight/ui/src/locales/zh-cn.json new file mode 100644 index 00000000..5c4eceaa --- /dev/null +++ b/mindinsight/ui/src/locales/zh-cn.json @@ -0,0 +1,159 @@ +{ + "public": { + "netWorkError": "网络或后台错误，请检查。", + "browserWarning": "您当前的浏览器可能导致部分功能失效或不可使用，建议使用Chrome 65版本以上的浏览器。", + "timeout": "超时，请重新请求。", + "noData": "暂无数据", + "reset": "重置", + "tagFilterPlaceHolder": "请输入需要的标签(支持正则表达式)", + "dataError": "获取到的数据异常", + "regIllegal": "请输入正确的检索条件" + }, + "symbols": { + "leftbracket": "(", + "rightbracket": ")" + }, + "header": { + "refreshData": "刷新数据", + "RefreshingData": "正在刷新数据", + "timeReload": "自动刷新", + "timeSecond": "秒" + }, + "summaryManage": { + "summaryList": "Summary列表", + "currentFolder": "当前文件夹：", + "sorting": "序号", + "summaryPath": "summary路径", + "createTime": "创建时间", + "updateTime": "更新时间", + "operation": "操作", + "viewDashboard": "查看训练看板", + "modelTraceback": "模型溯源", + "dataTraceback": "数据溯源" + }, + "modelTraceback": { + "summaryPath": "summary路径", + "trainSetPath": "训练集路径", + "testSetPath": "测试集路径", + "trainingSampleNum": "训练样本数量", + "testSampleNum": "测试样本数量", + "showAllData": "显示全量数据", + "network": "网络", + "optimizer": "优化器", + "lossFunc": "损失函数", + "learningRate": "学习率", + "modelSize": "模型大小", + "50540002": "Json数据解析失败", + "dataProcess": "数据处理" + }, + "dataTraceback": { + "details": "详情", + "key": "KEY", + "value": "VALUE", + "dataTraceTips": "该数据涉及合并操作" + }, + "trainingDashboard": { + "trainingDashboardTitle": "训练看板", + "calculationChart": "计算图", + "dataMap": "数据图", + "trainingScalar": "训练标量信息", + "samplingData": "数据抽样", + "imagesampleSwitch": "切换标签", + "invalidId": "无效的训练作业" + }, + "scalar": { + "titleText": "标量", + "tagSelectTitle": "标签选择", + "xAxisTitle": "水平轴", + "smoothness": "平滑度", + "step": "步骤", + "selectAll": "全选", + "relativeTime": "相对时间", + "absoluteTime": "绝对时间", + "fullScreen": "切换全屏", + "stepBack": "分步回退", + "openOrCloseSelection": "开启/关闭框选", + "toggleYaxisScale": "切换Y轴比例", + "charTipHeadName": "名称", + "charTipTagName": "标签", + "charTipHeadValue": "值", + "charSmoothedValue": "平滑值", + "comparison": "标量合成", + "compareCancel": "取消合成", + "open": "展开", + "close": "折叠", + "invalidData": "存在无效数据" + }, + + "images": { + "titleText": "图像", + "tagSelectTitle": "标签选择", + "selectAll": "全选", + "open": "展开", + "close": "折叠", + "step": "步骤：", + "setBright": "亮度调整", + "setContrast": "对比度调整" + }, + "dataMap": { + "titleText": "数据图可视" + }, + "graph": { + "titleText": "计算图可视", + "downloadPic": "下载", + "nodeInfo": "节点信息", + "legend": "图例", + "nameSpace": "命名空间", + "operatorNode": "算子节点", + "virtualNode": "虚拟节点", + "constantNode": "常量节点", + "virtualConstantNode": "虚拟常量节点", + "polymetric": "聚合节点", + "dataFlowEdge": "数据流向边", + "controllDepEdge": "控制依赖边", + "name": "名称", + "count": "子节点数", + "type": "类型", + "attr": "属性", + "inputs": "输入", + "outputs": "输出", + "outputs_i": "Outputs_i", + "controlDependencies": "控制边", + "searchLoading": "正在定位节点，请稍候。定位速度与节点数量有关，节点数量多时，速度会变慢。", + "queryLoading": "正在加载图，请稍候。", + "fullScreen": "全屏", + "tooManyNodes": "节点太多，打开失败", + "inputNodeName": "请输入节点名称", + "50540002": "当前训练作业已不存在，请选择其他训练作业进行查看。" + }, + "error": { + "200": "请求成功。", + "202": "接受请求。", + "400": "请求错误。", + "401": "鉴权失败。", + "403": "没有操作权限。", + "404": "找不到资源。", + "408": "请求超时。", + "409": "请求冲突。", + "429": "请求数太多。", + "500": "服务器内部错误。", + "503": "服务不可用。", + "50540000": "系统错误", + "50540001": "参数类型错误，请检查请求参数类型是否都符合要求", + "50540002": "参数值错误，请检查请求参数值是否都符合要求", + "50540003": "参数缺失必需的参数，请检查请求参数是否必填项都满足", + + "50545001": "API 路由资源不存在", + "50545002": "请求API的HTTP方法错误", + "50545006": "Summary日志路径无效", + "50545007": "Summary数据正在被加载，请等待Summary数据加载结束", + "50545009": "查询的节点不在图中", + + "50542082": "模型名称缺失", + "50542085": "模型名称不合法", + "50542215": "查询参数错误", + "50542216": "Summary log文件未找到", + "50542217": "Summary log路径错误", + "50542218": "筛选参数错误" + } +} diff --git a/mindinsight/ui/src/main.js b/mindinsight/ui/src/main.js new file mode 100644 index 00000000..3c83a48c --- /dev/null +++ b/mindinsight/ui/src/main.js @@ -0,0 +1,102 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Vue from 'vue'; +import App from './app.vue'; +import router from './router'; +import store from './store'; + +import ElementUI from 'element-ui'; +import './assets/css/element.css'; +import './assets/css/reset.scss'; +import i18n from './i18n'; + +Vue.use(ElementUI); + +Vue.prototype.$bus = new Vue(); + +// Route interception +router.beforeEach((to, from, next) => { + // cancel request + if (from.path !== '/') { + store.commit('clearToken'); + } + + // deter refresh + store.commit('setIsReload', false); + next(); +}); + +// forbidden showing production tip +Vue.config.productionTip = false; + +/** + * Check whether the input string contains special characters + * @param {String} strurl + * @return {Boolen} + */ +function justSql(strurl) { + const reA = /select|create|alert|update|delete|truncate/i; + const reB = /join|union|exec|insert|drop|count|'|"|;|>| { + new Vue({ + router, + store, + i18n, + render: (h) => h(App), + }).$mount('#app'); + }, 100); + } +}; diff --git a/mindinsight/ui/src/router.js b/mindinsight/ui/src/router.js new file mode 100644 index 00000000..1539a688 --- /dev/null +++ b/mindinsight/ui/src/router.js @@ -0,0 +1,67 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Vue from 'vue'; +import Router from 'vue-router'; +Vue.use(Router); + +export default new Router({ + base: process.env.BASE_URL, + routes: [ + // 404 matching + { + path: '*', + redirect: '/summary-manage', + }, + { + path: '/', + component: () => import('./views/train-manage/summary-manage.vue'), + redirect: '/summary-manage', + }, + { + path: '/summary-manage', + component: () => import('./views/train-manage/summary-manage.vue'), + }, + { + name: 'training-dashboard', + path: '/train-manage/training-dashboard', + component: () => import('./views/train-manage/training-dashboard.vue'), + }, + { + path: '/train-manage/scalar', + component: () => import('./views/train-manage/scalar.vue'), + }, + { + path: '/train-manage/image', + component: () => import('./views/train-manage/image.vue'), + }, + { + path: '/train-manage/graph', + component: () => import('./views/train-manage/graph.vue'), + }, + { + path: '/train-manage/data-map', + component: () => import('./views/train-manage/data-map.vue'), + }, + { + path: '/model-traceback', + component: () => import('./views/train-manage/model-traceback.vue'), + }, + { + path: '/data-traceback', + component: () => import('./views/train-manage/data-traceback.vue'), + }, + ], +}); diff --git a/mindinsight/ui/src/services/fetcher.js b/mindinsight/ui/src/services/fetcher.js new file mode 100644 index 00000000..c768efe6 --- /dev/null +++ b/mindinsight/ui/src/services/fetcher.js @@ -0,0 +1,107 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import router from '@/router'; +import store from '@/store'; +import i18n from '@/i18n'; +import axios from 'axios'; +import Vue from 'vue'; + +export {default} from 'axios'; +export const basePath = location.origin; + +axios.defaults.timeout = 30000; +axios.defaults.baseURL = basePath; +axios.interceptors.request.use( + function(config) { + config.headers['Pragma'] = 'no-cache'; + config.headers['Cache-Control'] = 'no-cache,no-store,must-revalidate'; + config.cancelToken = new axios.CancelToken((cancel) => { + store.commit('pushToken', { + cancelToken: cancel, + }); + }); + return config; + }, + function(error) { + return Promise.reject(error); + }, +); + +// Add a response interceptor +axios.interceptors.response.use( + function(response) { + return response; + }, + function(error) { + const errorData = i18n.messages[i18n.locale].error; + + // error returned by backend + if (error.response && error.response.data) { + if (error.response.data.error_code) { + // error code judgment + if (error.response.data.error_code.toString() === '50540002') { + if (router.currentRoute.path === '/train-manage/graph') { + const errorMsg = i18n.messages[i18n.locale].graph; + Vue.prototype.$message.error(errorMsg[error.response.data.error_code]); + } else if (error.config.headers.ignoreError + || router.currentRoute.path === '/train-manage/training-dashboard') { + return Promise.reject(error); + } else if (router.currentRoute.path === '/model-traceback') { + const errorMsg = i18n.messages[i18n.locale].modelTraceback; + Vue.prototype.$message.error(errorMsg[error.response.data.error_code]); + } else if (errorData[error.response.data.error_code]) { + Vue.prototype.$message.error(errorData[error.response.data.error_code]); + } + } else if (error.response.data.error_code.toString() === '50545006') { + return Promise.reject(error); + } else if (error.response.data.error_code.toString() === '50542216' && + router.currentRoute.path === '/train-manage/training-dashboard' + ) { + return Promise.reject(error); + } else if (errorData[error.response.data.error_code]) { + Vue.prototype.$message.error(errorData[error.response.data.error_code]); + } + // return error data + return Promise.reject(error); + } else { + if (error.response.data.status) { + if (errorData[error.response.data.status]) { + Vue.prototype.$message.error(errorData[error.response.data.status]); + } + return Promise.reject(error); + } + } + } else { + // error returned by browser + if (error.code === 'ECONNABORTED' && /^timeout/.test(error.message)) { + if (error.config.headers.ignoreError) { + return Promise.reject(error); + } + // timeout processing + Vue.prototype.$message.error(i18n.messages[i18n.locale].public.timeout); + return Promise.reject(error); + } else if (error.message === 'routeJump') { + // route jump + return false; + } else { + // show network error + Vue.prototype.$message.error(i18n.messages[i18n.locale].public.netWorkError); + return Promise.reject(error); + } + } + store.commit('setIsReload', false); + }, +); diff --git a/mindinsight/ui/src/services/request-service.js b/mindinsight/ui/src/services/request-service.js new file mode 100644 index 00000000..cca6ac99 --- /dev/null +++ b/mindinsight/ui/src/services/request-service.js @@ -0,0 +1,122 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import axios from './fetcher'; + +export default { + // query dataset graph data + queryDatasetGraph(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datasets/dataset_graph', + params: params, + }); + }, + + // query dataset lineage + getDatasetLineage() { + return axios({ + method: 'post', + url: 'v1/mindinsight/datasets/dataset_lineage', + }); + }, + + // query model versions + queryModelVersions(params) { + return axios({ + method: 'post', + url: 'v1/mindinsight/models/model_lineage', + data: params.body, + }); + }, + + // query summary list + querySummaryList(params) { + return axios({ + method: 'get', + url: '/v1/mindinsight/datavisual/train-jobs', + params: params, + }); + }, + + // query scalar sample + getScalarsSample(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/scalar/metadata', + params: params, + headers: { + ignoreError: true, + }, + }); + }, + + // query graph data + queryGraphData(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/graphs/nodes', + params: params, + }); + }, + + // search graph node + searchNodesNames(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/graphs/nodes/names', + params: params, + }); + }, + + // query the level of the node from the first layer based on node name + querySingleNode(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/graphs/single-node', + params: params, + }); + }, + + // query single train job list(image/scalar/graph) + getSingleTrainJob(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/single-job', + params: params, + }); + }, + + // query image meta data + getImageMetadatas(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/image/metadata', + params: params, + headers: { + ignoreError: true, + }, + }); + }, + + // query training job visualization plugins + getDatavisualPlugins(params) { + return axios({ + method: 'get', + url: 'v1/mindinsight/datavisual/plugins', + params: params, + }); + }, +}; diff --git a/mindinsight/ui/src/store.js b/mindinsight/ui/src/store.js new file mode 100644 index 00000000..4be65cd0 --- /dev/null +++ b/mindinsight/ui/src/store.js @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Vue from 'vue'; +import Vuex from 'vuex'; + +Vue.use(Vuex); + +export default new Vuex.Store({ + state: { + // cancel request token + cancelTokenArr: [], + // reload flag + isReload: false, + // Scheduled reload flag + isTimeReload: localStorage.isTimeReload === 'false' ? false : true, + // reload time + timeReloadValue: localStorage.timeReloadValue ? localStorage.timeReloadValue : 3, + + }, + mutations: { + // set cancelTokenArr + pushToken: (state, src) => { + state.cancelTokenArr.push(src.cancelToken); + }, + // clear cancelTokenArr + clearToken: (state) => { + state.cancelTokenArr.forEach((item) => { + item('routeJump'); + }); + state.cancelTokenArr = []; + }, + // set isReload + setIsReload: (state, val) => { + state.isReload = val; + }, + // set isTimeReload + setIsTimeReload: (state, val) => { + state.isTimeReload = val; + }, + setTimeReloadValue: (state, val) => { + state.timeReloadValue = val; + }, + + }, + actions: {}, +}); diff --git a/mindinsight/ui/src/views/train-manage/data-map.vue b/mindinsight/ui/src/views/train-manage/data-map.vue new file mode 100644 index 00000000..480e5841 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/data-map.vue @@ -0,0 +1,687 @@ + + + + + + diff --git a/mindinsight/ui/src/views/train-manage/data-traceback.vue b/mindinsight/ui/src/views/train-manage/data-traceback.vue new file mode 100644 index 00000000..a6302975 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/data-traceback.vue @@ -0,0 +1,979 @@ + + + + diff --git a/mindinsight/ui/src/views/train-manage/graph.vue b/mindinsight/ui/src/views/train-manage/graph.vue new file mode 100644 index 00000000..46c412c9 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/graph.vue @@ -0,0 +1,3160 @@ + + + + + + diff --git a/mindinsight/ui/src/views/train-manage/image.vue b/mindinsight/ui/src/views/train-manage/image.vue new file mode 100644 index 00000000..9fdb29ac --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/image.vue @@ -0,0 +1,1247 @@ + + + + + + diff --git a/mindinsight/ui/src/views/train-manage/model-traceback.vue b/mindinsight/ui/src/views/train-manage/model-traceback.vue new file mode 100644 index 00000000..fd2ed0d4 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/model-traceback.vue @@ -0,0 +1,839 @@ + + + + diff --git a/mindinsight/ui/src/views/train-manage/scalar-button.vue b/mindinsight/ui/src/views/train-manage/scalar-button.vue new file mode 100644 index 00000000..a41318ff --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/scalar-button.vue @@ -0,0 +1,60 @@ + + + + diff --git a/mindinsight/ui/src/views/train-manage/scalar-compare.vue b/mindinsight/ui/src/views/train-manage/scalar-compare.vue new file mode 100644 index 00000000..4b36d17a --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/scalar-compare.vue @@ -0,0 +1,1045 @@ + + + + diff --git a/mindinsight/ui/src/views/train-manage/scalar.vue b/mindinsight/ui/src/views/train-manage/scalar.vue new file mode 100644 index 00000000..01a49469 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/scalar.vue @@ -0,0 +1,2125 @@ + + + + diff --git a/mindinsight/ui/src/views/train-manage/summary-manage.vue b/mindinsight/ui/src/views/train-manage/summary-manage.vue new file mode 100644 index 00000000..001bb857 --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/summary-manage.vue @@ -0,0 +1,216 @@ + + + + + + diff --git a/mindinsight/ui/src/views/train-manage/training-dashboard.vue b/mindinsight/ui/src/views/train-manage/training-dashboard.vue new file mode 100644 index 00000000..e735762e --- /dev/null +++ b/mindinsight/ui/src/views/train-manage/training-dashboard.vue @@ -0,0 +1,1575 @@ + + + + + + diff --git a/mindinsight/ui/vue.config.js b/mindinsight/ui/vue.config.js new file mode 100644 index 00000000..6e51793c --- /dev/null +++ b/mindinsight/ui/vue.config.js @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const path = require('path'); + +/** + * @param {String} dir + * @return {String} + */ +function resolve(dir) { + return path.join(__dirname, dir); +} + +module.exports = { + publicPath: process.env.NODE_ENV === 'production' ? './' : '/', + outputDir: 'dist', + assetsDir: 'static', + + // map + productionSourceMap: false, + + configureWebpack: { + devtool: 'source-map', + }, + + chainWebpack: (config) => { + config.resolve.alias.set('@', resolve('src')); + + config.plugins.delete('preload'); + config.plugins.delete('prefetch'); + config.module + .rule('element-ui') + .test(/element-ui.src.*?js$/) + .use('babel') + .loader('babel-loader') + .end(); + }, + + devServer: { + port: 8085, + disableHostCheck: true, + }, + + pluginOptions: { + i18n: { + locale: 'zh-cn', + fallbackLocale: 'zh-cn', + localeDir: 'locales', + enableInSFC: true, + }, + }, + css: { + loaderOptions: { + sass: { + prependData: ` + @import "@/assets/css/variable.scss"; + `, + }, + }, + }, +}; diff --git a/mindinsight/utils/__init__.py b/mindinsight/utils/__init__.py new file mode 100644 index 00000000..8b70b66f --- /dev/null +++ b/mindinsight/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/utils/command.py b/mindinsight/utils/command.py new file mode 100644 index 00000000..e47f0558 --- /dev/null +++ b/mindinsight/utils/command.py @@ -0,0 +1,128 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Command module.""" + +import sys +import os +import argparse +from importlib import import_module + +import mindinsight +from mindinsight.utils.log import setup_logger +from mindinsight.utils.exceptions import MindInsightException + + +class BaseCommand: + """Base command class.""" + + name = '' + description = '' + + logger = None + + def add_arguments(self, parser): + """ + Add arguments to parser. + + Args: + parser (ArgumentParser): specify parser to which arguments are added. + """ + + def update_settings(self, args): + """ + Update settings. + + Args: + args (Namespace): parsed arguments to hold customized parameters. + """ + + def run(self, args): + """ + Implementation of command logic. + + Args: + args (Namespace): parsed arguments to hold customized parameters. + """ + raise NotImplementedError('subclasses of BaseCommand must provide a run() method') + + def invoke(self, args): + """ + Invocation of command. + + Args: + args (Namespace): parsed arguments to hold customized parameters. + """ + try: + self.update_settings(args) + except MindInsightException as error: + print(error.message) + else: + self.logger = setup_logger('scripts', self.name) + self.run(args) + + +def main(): + """Entry point for mindinsight CLI.""" + + if (sys.version_info.major, sys.version_info.minor) < (3, 7): + print('Python version should be at least 3.7') + return + + permissions = os.R_OK | os.W_OK | os.X_OK + + # set umask to 0o077 + os.umask(permissions << 3 | permissions) + + parser = argparse.ArgumentParser( + prog='mindinsight', + description='MindInsight CLI entry point (version: {})'.format(mindinsight.__version__)) + + parser.add_argument( + '--version', + action='version', + version='%(prog)s ({})'.format(mindinsight.__version__)) + + subparsers = parser.add_subparsers( + dest='cli', + title='subcommands', + description='the following subcommands are supported', + ) + + commands = {} + scripts_path = os.path.realpath(os.path.join(__file__, os.pardir, os.pardir, 'scripts')) + files = os.listdir(scripts_path) + files.sort() + for file in files: + if file.startswith('_') or not file.endswith('.py'): + continue + + module = import_module('mindinsight.scripts.{}'.format(file[:-len('.py')])) + command_cls = getattr(module, 'Command', None) + if command_cls is None or not issubclass(command_cls, BaseCommand): + continue + + command = command_cls() + command_parser = subparsers.add_parser(command.name, help=command.description) + command.add_arguments(command_parser) + commands[command.name] = command + + argv = sys.argv[1:] + if not argv or argv[0] == 'help': + argv = ['-h'] + + args = parser.parse_args(argv) + cli = args.__dict__.pop('cli') + command = commands[cli] + command.invoke(args) diff --git a/mindinsight/utils/constant.py b/mindinsight/utils/constant.py new file mode 100644 index 00000000..b9eaca99 --- /dev/null +++ b/mindinsight/utils/constant.py @@ -0,0 +1,60 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Constant module.""" + + +from enum import Enum + + +class MindInsightModules(Enum): + """ + Enum definition for MindInsight error types. + + Note: + Each enum value, excluding GENERAL, has an Errors class name starting with the enum value + in Camel-Case referring to specific module. + """ + GENERAL = 0 + LINEAGEMGR = 2 + DATAVISUAL = 5 + + +class GeneralErrors(Enum): + """Enum definition for general errors.""" + UNKNOWN_ERROR = 0 + PARAM_TYPE_ERROR = 1 + PARAM_VALUE_ERROR = 2 + PARAM_MISSING_ERROR = 3 + PATH_NOT_EXISTS_ERROR = 4 + FILE_SYSTEM_PERMISSION_ERROR = 8 + PORT_NOT_AVAILABLE_ERROR = 9 + + +class LineageMgrErrors(Enum): + """Enum definition for lineage errors.""" + + +class DataVisualErrors(Enum): + """Enum definition for datavisual errors.""" + RESTFUL_API_NOT_EXIST = 1 + REQUEST_METHOD_NOT_ALLOWED = 2 + SUMMARY_LOG_CONTENT_INVALID = 3 + CRC_FAILED = 4 + TRAIN_JOB_NOT_EXIST = 5 + SUMMARY_LOG_PATH_INVALID = 6 + SUMMARY_LOG_IS_LOADING = 7 + SUMMARY_LOG_LOAD_FAILED = 8 + NODE_NOT_IN_GRAPH_ERROR = 9 + PATH_NOT_DIRECTORY_ERROR = 10 diff --git a/mindinsight/utils/exceptions.py b/mindinsight/utils/exceptions.py new file mode 100644 index 00000000..4b1624d2 --- /dev/null +++ b/mindinsight/utils/exceptions.py @@ -0,0 +1,178 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Exception module.""" + +from importlib import import_module +from mindinsight.utils.constant import MindInsightModules, GeneralErrors + + +class MindInsightException(Exception): + """ + Base class for MindInsight exception. + + Examples: + >>> raise MindInsightException(GeneralErrors.PATH_NOT_EXISTS_ERROR, 'path not exists') + >>> raise MindInsightException(DataVisualErrors.CUSTOMIZED_ERROR, 'datavisual error message') + """ + + RUNTIME = 1 + TYPE = 1 + LEVEL = 0 + SYSID = 42 + + def __init__(self, error, message, http_code=500): + """ + Initialization of MindInsightException. + + Args: + error (Enum): Error value for specified case. + message (str): Description for exception. + http_code (int): Http code for exception. Default is 500. + """ + if isinstance(message, str): + message = ' '.join(message.split()) + super(MindInsightException, self).__init__(message) + self.error = error + self.message = message + self.http_code = http_code + + def parse_module(self): + """ + Parse module according to error enum class. + + Note: + Each enum value, excluding GENERAL, has an Errors class name starting with the enum value + in Camel-Case referring to specific module. + + Returns: + Enum, module for specified error. + """ + + module = None + constant = import_module('mindinsight.utils.constant') + errors_names = [item for item in dir(constant) if item.endswith('Errors')] + for name in errors_names: + errors_cls = getattr(constant, name) + if isinstance(self.error, errors_cls): + key = name[:-len('Errors')].upper() + module = getattr(MindInsightModules, key, None) + break + + return module + + @property + def error_code(self): + """ + Transform exception no to MindInsight error code. + + code compose(4bytes): + runtime 2bits, type 2bits, level 3bits, sysid 8bits, modid 5bits, value 12bits. + + num = ((0xFF & runtime) << 30) \ + | ((0xFF & type) << 28) \ + | ((0xFF & level) << 25) \ + | ((0xFF & sysid) << 17) \ + | ((0xFF & modid) << 12) \ + | (0x0FFF & value) + + Returns: + str, Hex string representing the composed MindInsight error code. + """ + + module = self.parse_module() + if not module: + raise UnknownError('Unknown module for {}.'.format(self.error)) + + num = (((0xFF & self.RUNTIME) << 30) + | ((0xFF & self.TYPE) << 28) + | ((0xFF & self.LEVEL) << 25) + | ((0xFF & self.SYSID) << 17) + | ((0xFF & module.value) << 12) + | (0x0FFF & self.error.value)) + + return hex(num)[2:].zfill(8).upper() + + def __str__(self): + return '[{}] code: {}, msg: {}'.format(self.__class__.__name__, self.error_code, self.message) + + +class ParamValueError(MindInsightException): + """Request param value error.""" + def __init__(self, error_detail): + error_msg = 'Invalid parameter value. {}'.format(error_detail) + super(ParamValueError, self).__init__( + GeneralErrors.PARAM_VALUE_ERROR, + error_msg, + http_code=400) + + +class ParamTypeError(MindInsightException): + """Request param type error.""" + def __init__(self, param_name, expected_type): + error_msg = "Invalid parameter type. '{}' expect {} type.".format(param_name, expected_type) + super(ParamTypeError, self).__init__( + GeneralErrors.PARAM_TYPE_ERROR, + error_msg, + http_code=400) + + +class ParamMissError(MindInsightException): + """Missing param error.""" + def __init__(self, param_name): + error_msg = "Param missing. '{}' is required.".format(param_name) + super(ParamMissError, self).__init__( + GeneralErrors.PARAM_MISSING_ERROR, + error_msg, + http_code=400) + + +class PathNotExistError(MindInsightException): + """Raised when specified path do not exist.""" + def __init__(self, error_detail): + """Initialize PathNotExistError.""" + error_msg = 'Specified path does not exist. Detail: {}'.format(error_detail) + super(PathNotExistError, self).__init__( + GeneralErrors.PATH_NOT_EXISTS_ERROR, + error_msg, + http_code=400) + + +class FileSystemPermissionError(MindInsightException): + """Can not access file or dir.""" + def __init__(self, error_detail): + error_msg = 'File or dir access failed. Detail: {}'.format(error_detail) + super(FileSystemPermissionError, self).__init__( + GeneralErrors.FILE_SYSTEM_PERMISSION_ERROR, + error_msg, + http_code=400) + + +class PortNotAvailableError(MindInsightException): + """Port not available error..""" + def __init__(self, error_detail): + error_msg = 'Port not available error. Detail: {}'.format(error_detail) + super(PortNotAvailableError, self).__init__( + GeneralErrors.PORT_NOT_AVAILABLE_ERROR, + error_msg, + http_code=400) + + +class UnknownError(MindInsightException): + """Unknown error.""" + def __init__(self, error_msg): + super(UnknownError, self).__init__( + GeneralErrors.UNKNOWN_ERROR, + error_msg, + http_code=500) diff --git a/mindinsight/utils/hook.py b/mindinsight/utils/hook.py new file mode 100644 index 00000000..319ae331 --- /dev/null +++ b/mindinsight/utils/hook.py @@ -0,0 +1,165 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Hook module.""" + +import os +import threading +from importlib import import_module + +from mindinsight.conf import settings +from mindinsight.utils.exceptions import FileSystemPermissionError + + +class BaseHook: + """Base hook class.""" + + def register_secure_domains(self): + """Hook function to register secure domains.""" + return [] + + def register_startup_arguments(self, parser): + """ + Hook function to register startup arguments. + + Args: + parser (ArgumentParser): specify parser to which arguments are added. + """ + + def on_startup(self, logger): + """ + Hook function to on startup. + + Args: + logger (Logger): script logger of start command. + """ + + def on_shutdown(self, logger): + """ + Hook function to on shutdown. + + Args: + logger (Logger): script logger of stop command. + """ + + def on_init(self): + """Hook function to on init.""" + + +class HookUtils: + """ + Lock utilities. + + Examples: + >>> from mindinsight.utils.hook import HookUtils + >>> for hook in HookUtils.instance().hooks(): + >>> domains = hook.register_secure_domains() + >>> hook.register_startup_arguments(parser) + >>> hook.on_startup(logger) + >>> hook.on_shutdown(logger) + >>> hook.on_init() + """ + + _lock = threading.Lock() + _instance = None + + def __new__(cls, *args, **kwargs): + """Built-in __new__ function.""" + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + cls._instance.discover() + return cls._instance + + def discover(self): + """Discover hook instances.""" + self.__hooks = [] + mindinsight_path = os.path.join(__file__, os.pardir, os.pardir) + hook_path = os.path.realpath(os.path.join(mindinsight_path, 'common/hook')) + files = os.listdir(hook_path) + files.sort() + for file in files: + if file.startswith('_') or not file.endswith('.py'): + continue + hook_name = file[:-len('.py')] + hook_module = import_module('mindinsight.common.hook.{}'.format(hook_name)) + hook_cls = getattr(hook_module, 'Hook', None) + if hook_cls is not None and issubclass(hook_cls, BaseHook): + self.__hooks.append(hook_cls()) + + def hooks(self): + """ + Return list of hook instances. + + Returns: + list, list of hook instances. + """ + return self.__hooks + + @classmethod + def instance(cls): + """ + Produce singleton instance of HookUtils. + + Returns: + HookUtils, singleton instance of HookUtils. + """ + if cls._instance is None: + cls._instance = cls() + return cls._instance + + +def init(workspace='', config='', **kwargs): + """ + Init MindInsight context. + + Args: + workspace (str): specify mindinsight workspace, default is ''. + config (str): specify mindinsight config file, default is ''. + + Raises: + FileSystemPermissionError, if workspace is not allowed to access or available. + """ + permissions = os.R_OK | os.W_OK | os.X_OK + + # set umask to 0o077 + os.umask(permissions << 3 | permissions) + + # assign argument values into environment + if workspace: + kwargs['workspace'] = workspace + + if config: + kwargs['config'] = config + + for key, value in kwargs.items(): + variable = 'MINDINSIGHT_{}'.format(key.upper()) + os.environ[variable] = str(value) + + settings.refresh() + + if os.path.exists(settings.WORKSPACE): + if not os.access(settings.WORKSPACE, permissions): + raise FileSystemPermissionError('Workspace {} not allowed to access'.format(workspace)) + else: + try: + mode = permissions << 6 + os.makedirs(settings.WORKSPACE, mode=mode, exist_ok=True) + except OSError: + # race condition or priority problem + raise FileSystemPermissionError('Workspace {} not available'.format(workspace)) + + for hook in HookUtils.instance().hooks(): + hook.on_init() diff --git a/mindinsight/utils/log.py b/mindinsight/utils/log.py new file mode 100644 index 00000000..90b3f409 --- /dev/null +++ b/mindinsight/utils/log.py @@ -0,0 +1,226 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log module.""" + +import sys +import os +import stat +import time +import fcntl +import logging +from logging.handlers import RotatingFileHandler + +from mindinsight.conf import settings +from mindinsight.utils.exceptions import MindInsightException +from mindinsight.utils.constant import GeneralErrors + + +class MultiCompatibleRotatingFileHandler(RotatingFileHandler): + """Inherit RotatingFileHandler for multiprocess compatibility.""" + + def rolling_rename(self): + """Rolling rename log files.""" + for i in range(self.backupCount - 1, 0, -1): + sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i)) + dfn = self.rotation_filename("%s.%d" % (self.baseFilename, i + 1)) + if os.path.exists(sfn): + if os.path.exists(dfn): + os.remove(dfn) + os.chmod(sfn, stat.S_IREAD) + os.rename(sfn, dfn) + + def doRollover(self): + """Do a rollover, as described in __init__().""" + if self.stream: + self.stream.close() + self.stream = None + + # Attain an exclusive lock with bloking mode by `fcntl` module. + with open(self.baseFilename, 'a') as file_pointer: + fcntl.lockf(file_pointer.fileno(), fcntl.LOCK_EX) + + if self.backupCount > 0: + self.rolling_rename() + + dfn = self.rotation_filename(self.baseFilename + ".1") + if os.path.exists(dfn): + os.remove(dfn) + + os.chmod(self.baseFilename, stat.S_IREAD) + self.rotate(self.baseFilename, dfn) + + with open(self.baseFilename, 'a'): + os.chmod(self.baseFilename, stat.S_IREAD | stat.S_IWRITE) + + if not self.delay: + self.stream = self._open() + + def _open(self): + """Open the current base file with the (original) mode and encoding.""" + new_log = open(self.baseFilename, self.mode, encoding=self.encoding) + os.chmod(self.baseFilename, stat.S_IREAD | stat.S_IWRITE) + return new_log + + +class MindInsightFormatter(logging.Formatter): + """ + MindInsight formatter. + """ + + def __init__(self, sub_module, fmt=None, **kwargs): + """ + Initialization of SlogFormatter. + + Args: + sub_module (str): Sub module name, type is string. + fmt (str): Specified format pattern, type is string. + + Returns: + Formatter, instance of SlogFormatter. + """ + super(MindInsightFormatter, self).__init__(fmt=fmt, **kwargs) + self.sub_module = sub_module.upper() + + def formatTime(self, record, datefmt=None): + """ + Overwrite for uniform format %Y-%m-%d-%H:%M:%S.SSS.SSS + + Args: + record (LogRecord): Log record. + datefmt (str): Date format, type is string. + + Returns: + str, formatted timestamp, type is string. + """ + created_time = self.converter(record.created) + if datefmt: + return time.strftime(datefmt, created_time) + + timestamp = time.strftime('%Y-%m-%d-%H:%M:%S', created_time) + msecs = str(round(record.msecs * 1000)).zfill(6) + return '{}.{}.{}'.format(timestamp, msecs[:3], msecs[3:]) + + def formatMessage(self, record): + """Escape the message before format.""" + record.message = ' '.join(record.message.split()) + return super().formatMessage(record) + + def format(self, record): + """ + Apply log format with specified pattern. + + Args: + record (str): Format pattern, type is string. + + Returns: + str, formatted log content according to format pattern, type if string. + """ + record.filepath = record.pathname[__file__.rfind('mindinsight'):] + record.sub_module = self.sub_module + return super().format(record) + + +def get_logger(sub_module, log_name): + """ + Get logger by name and sub module. + + Args: + sub_module (str): Sub module name, type is string. + log_name (str): Log file name, type is string. + + Returns: + Logger, logger instance named by sub_module and log_name. + """ + return logging.getLogger(name='{}.{}'.format(sub_module, log_name)) + + +def setup_logger(sub_module, log_name, console=False, logfile=True, **kwargs): + """ + Setup logger with sub module name and log file name. + + Args: + sub_module (str): Sub module name, also for sub directory under logroot. + log_name (str): Log name, also for log filename. + console (bool): Whether to output log to stdout. Default: False. + logfile (bool): Whether to output log to disk. Default: True. + level (Enum): Log level. Default: INFO. + formatter (str): Log format. + propagate (bool): Whether to enable propagate feature. Default: False. + maxBytes (int): Rotating max bytes. Default: 50M. + backupCount (int): Rotating backup count. Default: 30. + + Returns: + Logger, well-configured logger instance. + + Examples: + >>> from mindinsight.utils.log import setup_logger + >>> logger = setup_logger('datavisual', 'flask.request', level=logging.DEBUG) + + >>> from mindinsight.utils.log import get_logger + >>> logger = get_logger('datavisual', 'flask.request') + + >>> import logging + >>> logger = logging.getLogger('datavisual.flask.request') + """ + + logger = get_logger(sub_module, log_name) + if logger.hasHandlers(): + return logger + + level = kwargs.get('level', settings.LOG_LEVEL) + formatter = kwargs.get('formatter', None) + propagate = kwargs.get('propagate', False) + + logger.setLevel(level) + logger.propagate = propagate + + if not formatter: + formatter = settings.LOG_FORMAT + + if console: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.formatter = MindInsightFormatter(sub_module, formatter) + logger.addHandler(console_handler) + + if logfile: + max_bytes = kwargs.get('maxBytes', settings.LOG_ROTATING_MAXBYTES) + + if not isinstance(max_bytes, int) or not max_bytes > 0: + raise MindInsightException(GeneralErrors.PARAM_VALUE_ERROR, + 'maxBytes should be int type and > 0.') + + backup_count = kwargs.get('backupCount', + settings.LOG_ROTATING_BACKUPCOUNT) + + if not isinstance(backup_count, int) or not backup_count > 0: + raise MindInsightException(GeneralErrors.PARAM_VALUE_ERROR, + 'backupCount should be int type and > 0.') + + logfile_dir = os.path.join(settings.WORKSPACE, 'log', sub_module) + + permissions = os.R_OK | os.W_OK | os.X_OK + mode = permissions << 6 + os.makedirs(logfile_dir, mode=mode, exist_ok=True) + + logfile_handler = MultiCompatibleRotatingFileHandler( + filename=os.path.join(logfile_dir, '{}.log'.format(log_name)), + maxBytes=max_bytes, + backupCount=backup_count, + encoding='utf8' + ) + logfile_handler.formatter = MindInsightFormatter(sub_module, formatter) + logger.addHandler(logfile_handler) + + return logger diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..43e3fc8e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +attrdict>=2.0.1 +Click>=7.0 +Flask>=1.1.1 +Flask-Cors>=3.0.8 +gunicorn>=19.9.0 +itsdangerous>=1.1.0 +Jinja2>=2.10.1 +MarkupSafe>=1.1.1 +marshmallow>=2.19.2 +numpy>=1.17.0 +protobuf>=3.8.0 +psutil>=5.6.1 +six>=1.12.0 +Werkzeug>=1.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..af15176d --- /dev/null +++ b/setup.py @@ -0,0 +1,190 @@ +# Copyright 2019 Huawei Technologies Co., Ltd.All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Setup.""" + +import sys +import os +import shutil +import stat +import platform +import shlex +import subprocess +import types +from importlib import import_module +from setuptools import setup +from setuptools.command.egg_info import egg_info +from setuptools.command.build_py import build_py +from setuptools.command.install import install + + +def get_version(): + """Get version.""" + machinery = import_module('importlib.machinery') + version_path = os.path.join(os.path.dirname(__file__), 'mindinsight', '_version.py') + module_name = '__mindinsightversion__' + version_module = types.ModuleType(module_name) + loader = machinery.SourceFileLoader(module_name, version_path) + loader.exec_module(version_module) + return version_module.VERSION + + +def get_os(): + """Get OS.""" + os_system = platform.system().lower() + return os_system + + +def get_description(): + """Get description.""" + os_info = get_os() + cpu_info = platform.machine() + + cmd = "git log --format='[sha1]:%h, [branch]:%d' -1" + process = subprocess.Popen( + shlex.split(cmd), + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, _ = process.communicate() + if not process.returncode: + git_version = stdout.decode() + return 'mindinsight platform: %s, cpu: %s, git version: %s' % (os_info, cpu_info, git_version) + + return 'mindinsight platform: %s, cpu: %s' % (os_info, cpu_info) + + +def get_install_requires(): + """Get install requirements.""" + with open('requirements.txt') as file: + return file.read().splitlines() + + +def update_permissions(path): + """ + Update permissions. + + Args: + path (str): Target directory path. + """ + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + dir_fullpath = os.path.join(dirpath, dirname) + os.chmod(dir_fullpath, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP) + for filename in filenames: + file_fullpath = os.path.join(dirpath, filename) + os.chmod(file_fullpath, stat.S_IREAD) + + +def run_script(script): + """ + Run script. + + Args: + script (str): Target script file path. + """ + cmd = '/bin/bash {}'.format(script) + process = subprocess.Popen( + shlex.split(cmd), + shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + sys.stdout.write(line.decode()) + + if process.returncode: + sys.exit(1) + + +class EggInfo(egg_info): + """Egg info.""" + def run(self): + self.build_dependencies() + + egg_info_dir = os.path.join(os.path.dirname(__file__), 'mindinsight.egg-info') + shutil.rmtree(egg_info_dir, ignore_errors=True) + + super().run() + + update_permissions(egg_info_dir) + + def build_dependencies(self): + build_dir = os.path.join(os.path.dirname(__file__), 'build') + + sys.stdout.write('building crc32 ...\n') + crc32_script = os.path.join(build_dir, 'scripts', 'crc32.sh') + run_script(crc32_script) + + sys.stdout.write('building ui ...\n') + ui_script = os.path.join(build_dir, 'scripts', 'ui.sh') + run_script(ui_script) + + +class BuildPy(build_py): + """Build py files.""" + def run(self): + mindinsight_lib_dir = os.path.join(os.path.dirname(__file__), 'build', 'lib', 'mindinsight') + shutil.rmtree(mindinsight_lib_dir, ignore_errors=True) + + super().run() + + update_permissions(mindinsight_lib_dir) + + +class Install(install): + """Install.""" + def run(self): + super().run() + + if sys.argv[-1] == 'install': + pip = import_module('pip') + mindinsight_dir = os.path.join(os.path.dirname(pip.__path__[0]), 'mindinsight') + update_permissions(mindinsight_dir) + + +if __name__ == '__main__': + version_info = sys.version_info + if (version_info.major, version_info.minor) < (3, 7): + sys.stderr.write('Python version should be at least 3.7\r\n') + sys.exit(1) + + setup(name='mindinsight', + version=get_version(), + author='MindInsight Team', + description=get_description(), + license='Apache 2.0', + keywords='mindinsight', + install_requires=get_install_requires(), + packages=['mindinsight'], + platforms=[get_os()], + include_package_data=True, + cmdclass={ + 'egg_info': EggInfo, + 'build_py': BuildPy, + 'install': Install, + }, + entry_points={ + 'console_scripts': [ + 'mindinsight=mindinsight.utils.command:main', + ], + }) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/__init__.py b/tests/st/func/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/__init__.py b/tests/st/func/datavisual/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/conftest.py b/tests/st/func/datavisual/conftest.py new file mode 100644 index 00000000..6a28c301 --- /dev/null +++ b/tests/st/func/datavisual/conftest.py @@ -0,0 +1,109 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Description: This file is used for some common util. +""" +import os +import shutil +from unittest.mock import Mock +import pytest +from flask import Response + +from tests.st.func.datavisual import constants +from tests.st.func.datavisual.utils.log_operations import LogOperations +from tests.st.func.datavisual.utils.utils import check_loading_done +from tests.st.func.datavisual.utils import globals as gbl +from mindinsight.conf import settings +from mindinsight.datavisual.data_transform import data_manager +from mindinsight.datavisual.data_transform.data_manager import DataManager +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.datavisual.data_transform.loader_generators.loader_generator import MAX_DATA_LOADER_SIZE +from mindinsight.datavisual.utils import tools + +summaries_metadata = None +mock_data_manager = None +summary_base_dir = constants.SUMMARY_BASE_DIR + + +@pytest.fixture(autouse=True) +def set_summary_base_dir(monkeypatch): + """Mock settings.SUMMARY_BASE_DIR.""" + monkeypatch.setattr(settings, 'SUMMARY_BASE_DIR', summary_base_dir) + + +@pytest.fixture(scope="session") +def init_summary_logs(): + """Init summary logs.""" + try: + if os.path.exists(summary_base_dir): + shutil.rmtree(summary_base_dir) + permissions = os.R_OK | os.W_OK | os.X_OK + mode = permissions << 6 + if not os.path.exists(summary_base_dir): + os.mkdir(summary_base_dir, mode=mode) + global summaries_metadata, mock_data_manager + log_operations = LogOperations() + summaries_metadata = log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_FIRST) + mock_data_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) + mock_data_manager.start_load_data(reload_interval=0) + check_loading_done(mock_data_manager) + + summaries_metadata.update(log_operations.create_summary_logs( + summary_base_dir, constants.SUMMARY_DIR_NUM_SECOND, constants.SUMMARY_DIR_NUM_FIRST)) + summaries_metadata.update(log_operations.create_multiple_logs( + summary_base_dir, constants.MULTIPLE_DIR_NAME, constants.MULTIPLE_LOG_NUM)) + summaries_metadata.update(log_operations.create_reservoir_log( + summary_base_dir, constants.RESERVOIR_DIR_NAME, constants.RESERVOIR_STEP_NUM)) + mock_data_manager.start_load_data(reload_interval=0) + + # Sleep 1 sec to make sure the status of mock_data_manager changed to LOADING. + check_loading_done(mock_data_manager, first_sleep_time=1) + + # Maximum number of loads is `MAX_DATA_LOADER_SIZE`. + for i in range(len(summaries_metadata) - MAX_DATA_LOADER_SIZE): + summaries_metadata.pop("./%s%d" % (constants.SUMMARY_PREFIX, i)) + + yield + finally: + if os.path.exists(summary_base_dir): + shutil.rmtree(summary_base_dir) + + +@pytest.fixture(autouse=True) +def populate_globals(): + """Populate globals.""" + gbl.summaries_metadata = summaries_metadata + gbl.mock_data_manager = mock_data_manager + + +@pytest.fixture +def client(): + """This fixture is flask client.""" + + gbl.mock_data_manager.start_load_data(reload_interval=0) + check_loading_done(gbl.mock_data_manager) + + data_manager.DATA_MANAGER = gbl.mock_data_manager + + packages = ["mindinsight.backend.datavisual"] + + mock_obj = Mock(return_value=packages) + tools.find_app_package = mock_obj + + from mindinsight.backend.application import APP + APP.response_class = Response + app_client = APP.test_client() + + yield app_client diff --git a/tests/st/func/datavisual/constants.py b/tests/st/func/datavisual/constants.py new file mode 100644 index 00000000..04f43fc8 --- /dev/null +++ b/tests/st/func/datavisual/constants.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Constants for st.""" +import tempfile + +SUMMARY_BASE_DIR = tempfile.NamedTemporaryFile().name +SUMMARY_PREFIX = "summary" + +SUMMARY_DIR_NUM_FIRST = 5 +SUMMARY_DIR_NUM_SECOND = 11 + +RESERVOIR_DIR_NAME = "reservoir_dir" +RESERVOIR_TRAIN_ID = "./%s" % RESERVOIR_DIR_NAME +RESERVOIR_STEP_NUM = 15 +RESERVOIR_DIR_NUM = 1 + +MULTIPLE_DIR_NAME = "multiple_dir" +MULTIPLE_TRAIN_ID = "./%s" % MULTIPLE_DIR_NAME +MULTIPLE_LOG_NUM = 3 +MULTIPLE_DIR_NUM = 1 + +# Please make sure SUMMARY_DIR_NUM is greater than `MAX_DATA_LOADER_SIZE`. +# Mainly used to test caching. +SUMMARY_DIR_NUM = SUMMARY_DIR_NUM_FIRST\ + + SUMMARY_DIR_NUM_SECOND\ + + RESERVOIR_DIR_NUM\ + + MULTIPLE_DIR_NUM diff --git a/tests/st/func/datavisual/graph/__init__.py b/tests/st/func/datavisual/graph/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/graph/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result1.json b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result1.json new file mode 100644 index 00000000..e9c54d67 --- /dev/null +++ b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result1.json @@ -0,0 +1 @@ +{"nodes":[{"attr":{},"input":{},"name":"Default/conv1-Conv2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":1,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1-BatchNorm2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":14,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":20,"type":"name_scope"}]} diff --git a/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result2.json b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result2.json new file mode 100644 index 00000000..adfe1106 --- /dev/null +++ b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result2.json @@ -0,0 +1 @@ +{"nodes":[{"attr":{},"input":{},"name":"Default","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":3,"type":"name_scope"}]} diff --git a/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result3.json b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result3.json new file mode 100644 index 00000000..c178229e --- /dev/null +++ b/tests/st/func/datavisual/graph/graph_results/test_query_nodes_success_result3.json @@ -0,0 +1 @@ +{"nodes":[{"attr":{},"input":{"Default/bn1/Add50":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/conv1.weight":{"edge_type":"data","scope":"polymeric_scope","shape":[64,3,7,7]},"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x10":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x2":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x3":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x4":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x5":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x6":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x7":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x8":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x9":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape1","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add50":{"edge_type":"data"},"Default/bn1/conv1.weight":{"edge_type":"data"},"Default/bn1/x":{"edge_type":"data"},"Default/bn1/x1":{"edge_type":"data"},"Default/bn1/x10":{"edge_type":"data"},"Default/bn1/x2":{"edge_type":"data"},"Default/bn1/x3":{"edge_type":"data"},"Default/bn1/x4":{"edge_type":"data"},"Default/bn1/x5":{"edge_type":"data"},"Default/bn1/x6":{"edge_type":"data"},"Default/bn1/x7":{"edge_type":"data"},"Default/bn1/x8":{"edge_type":"data"},"Default/bn1/x9":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add51":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape2","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add51":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add52":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape3","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add52":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add53":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape4","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add53":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add54":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape5","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add54":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape6","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape7","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape8","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape9","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape10","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape11","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape12","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"}]} diff --git a/tests/st/func/datavisual/graph/graph_results/test_query_single_node_success_result1.json b/tests/st/func/datavisual/graph/graph_results/test_query_single_node_success_result1.json new file mode 100644 index 00000000..f23d28c7 --- /dev/null +++ b/tests/st/func/datavisual/graph/graph_results/test_query_single_node_success_result1.json @@ -0,0 +1 @@ +{"children":{"children":{"children":{"children":{},"nodes":[{"attr":{},"input":{"Default/bn1/Add50":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/conv1.weight":{"edge_type":"data","scope":"polymeric_scope","shape":[64,3,7,7]},"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x10":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x2":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x3":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x4":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x5":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x6":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x7":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x8":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x9":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape1","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add50":{"edge_type":"data"},"Default/bn1/conv1.weight":{"edge_type":"data"},"Default/bn1/x":{"edge_type":"data"},"Default/bn1/x1":{"edge_type":"data"},"Default/bn1/x10":{"edge_type":"data"},"Default/bn1/x2":{"edge_type":"data"},"Default/bn1/x3":{"edge_type":"data"},"Default/bn1/x4":{"edge_type":"data"},"Default/bn1/x5":{"edge_type":"data"},"Default/bn1/x6":{"edge_type":"data"},"Default/bn1/x7":{"edge_type":"data"},"Default/bn1/x8":{"edge_type":"data"},"Default/bn1/x9":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add51":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape2","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add51":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add52":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape3","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add52":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add53":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape4","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add53":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/Add54":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"name":"Default/bn1/Reshape5","output":{},"output_i":0,"polymeric_input":{"Default/bn1/Add54":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape6","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape7","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape8","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape9","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape10","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape11","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"},{"attr":{},"input":{"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape12","output":{},"output_i":0,"polymeric_input":{"Default/bn1/x":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"Default/bn1/Reshape_1_[12]","subnode_count":0,"type":"Reshape"}],"scope_name":"Default/bn1/Reshape_1_[12]"},"nodes":[{"attr":{},"input":{"Default/bn1/x11":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Add50","output":{"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"output_i":0,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Add"},{"attr":{},"input":{"Default/bn1/x11":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Add51","output":{"Default/bn1/Reshape2":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"output_i":0,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Add"},{"attr":{},"input":{"Default/bn1/x11":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Add52","output":{"Default/bn1/Reshape3":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"output_i":0,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Add"},{"attr":{},"input":{"Default/bn1/cst13":{"edge_type":"data","scope":"name_scope","shape":[]},"Default/bn1/x":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x1":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x10":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x11":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x2":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x3":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x4":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x5":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x6":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x7":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x8":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/x9":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Add53","output":{"Default/bn1/Reshape4":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"output_i":0,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Add"},{"attr":{},"input":{"Default/bn1/x11":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Add54","output":{"Default/bn1/Reshape5":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]}},"output_i":0,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Add"},{"attr":{},"input":{},"name":"Default/bn1/x","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape10":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape11":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape12":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape6":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape7":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape8":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/Reshape9":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x1","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x2","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x3","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x4","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x5","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x6","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x7","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x8","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x9","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/x10","output":{"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{},"name":"Default/bn1/conv1.weight","output":{"Default/bn1/Reshape1":{"edge_type":"data","scope":"polymeric_scope","shape":[64,3,7,7]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{"Default/bn1/Reshape_1_[12]":{"edge_type":"data"}},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{"cst13":"dtype: DT_INT32\nint_val: 0\n"},"input":{},"name":"Default/bn1/cst13","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":0,"type":"Const"},{"attr":{},"input":{},"name":"Default/bn1/x11","output":{"Default/bn1/Add50":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Add51":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Add52":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Add53":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]},"Default/bn1/Add54":{"edge_type":"data","scope":"name_scope","shape":[1,3,224,224]}},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":0,"type":"Parameter"},{"attr":{},"input":{"Default/bn1/Add50":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/Add51":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/Add52":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/Add53":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/Add54":{"edge_type":"data","scope":"polymeric_scope","shape":[1,1024,14,14]},"Default/bn1/conv1.weight":{"edge_type":"data","scope":"polymeric_scope","shape":[64,3,7,7]},"Default/bn1/x":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x1":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x10":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x2":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x3":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x4":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x5":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x6":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x7":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x8":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]},"Default/bn1/x9":{"edge_type":"data","scope":"polymeric_scope","shape":[1,3,224,224]}},"name":"Default/bn1/Reshape_1_[12]","output":{},"output_i":-1,"polymeric_input":{"Default/bn1/Add50":{"edge_type":"data"},"Default/bn1/Add51":{"edge_type":"data"},"Default/bn1/Add52":{"edge_type":"data"},"Default/bn1/Add53":{"edge_type":"data"},"Default/bn1/Add54":{"edge_type":"data"},"Default/bn1/conv1.weight":{"edge_type":"data"},"Default/bn1/x":{"edge_type":"data"},"Default/bn1/x1":{"edge_type":"data"},"Default/bn1/x10":{"edge_type":"data"},"Default/bn1/x2":{"edge_type":"data"},"Default/bn1/x3":{"edge_type":"data"},"Default/bn1/x4":{"edge_type":"data"},"Default/bn1/x5":{"edge_type":"data"},"Default/bn1/x6":{"edge_type":"data"},"Default/bn1/x7":{"edge_type":"data"},"Default/bn1/x8":{"edge_type":"data"},"Default/bn1/x9":{"edge_type":"data"}},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":12,"type":"polymeric_scope"}],"scope_name":"Default/bn1"},"nodes":[{"attr":{},"input":{},"name":"Default/conv1-Conv2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":1,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1-BatchNorm2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":14,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":20,"type":"name_scope"}],"scope_name":"Default"},"nodes":[{"attr":{},"input":{},"name":"Default","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":3,"type":"name_scope"}],"scope_name":""} diff --git a/tests/st/func/datavisual/graph/graph_results/test_search_nodes_success_result1.json b/tests/st/func/datavisual/graph/graph_results/test_search_nodes_success_result1.json new file mode 100644 index 00000000..7358c792 --- /dev/null +++ b/tests/st/func/datavisual/graph/graph_results/test_search_nodes_success_result1.json @@ -0,0 +1 @@ +{"names":["Default/bn1/Reshape1","Default/bn1/Reshape10","Default/bn1/Reshape11","Default/bn1/Reshape12","Default/bn1/Reshape2","Default/bn1/Reshape3","Default/bn1/Reshape4","Default/bn1/Reshape5","Default/bn1/Reshape6","Default/bn1/Reshape7","Default/bn1/Reshape8","Default/bn1/Reshape9","Default/bn1/Reshape_1_[12]"]} diff --git a/tests/st/func/datavisual/graph/test_query_nodes_restful_api.py b/tests/st/func/datavisual/graph/test_query_nodes_restful_api.py new file mode 100644 index 00000000..63520fed --- /dev/null +++ b/tests/st/func/datavisual/graph/test_query_nodes_restful_api.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test query nodes restful api. +Usage: + pytest tests/st/func/datavisual +""" +import os +import json +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +BASE_URL = '/v1/mindinsight/datavisual/graphs/nodes' + + +class TestQueryNodes: + """Test query nodes restful APIs.""" + + graph_results_dir = os.path.join(os.path.dirname(__file__), 'graph_results') + + def compare_result_with_file(self, result, filename): + """Compare result with file which contain the expected results.""" + with open(os.path.join(self.graph_results_dir, filename), 'r') as fp: + expected_results = json.load(fp) + assert result == expected_results + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("node_name, node_type, result_file", [ + ('', "name_scope", "test_query_nodes_success_result2.json"), + ("Default", "name_scope", "test_query_nodes_success_result1.json"), + ("Default/bn1/Reshape_1_[12]", "polymeric_scope", "test_query_nodes_success_result3.json") + ]) + def test_query_namescope_success(self, client, node_name, node_type, result_file): + """Query the name scope node.""" + train_id = gbl.get_train_ids()[0] + + if node_name: + params = dict(train_id=train_id, + type=node_type, + name=node_name) + else: + params = dict(train_id=train_id, + type=node_type) + url = get_url(BASE_URL, params) + response = client.get(url) + assert response.status_code == 200 + self.compare_result_with_file(response.get_json(), result_file) diff --git a/tests/st/func/datavisual/graph/test_query_single_nodes_restful_api.py b/tests/st/func/datavisual/graph/test_query_single_nodes_restful_api.py new file mode 100644 index 00000000..d1c893e7 --- /dev/null +++ b/tests/st/func/datavisual/graph/test_query_single_nodes_restful_api.py @@ -0,0 +1,62 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test search nodes restful api. +Usage: + pytest tests/st/func/datavisual +""" +import os +import json + +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +BASE_URL = '/v1/mindinsight/datavisual/graphs/single-node' + + +class TestQuerySingleNode: + """Test query single node restful APIs.""" + + graph_results_dir = os.path.join(os.path.dirname(__file__), 'graph_results') + + def compare_result_with_file(self, result, filename): + """Compare result with file which contain the expected results.""" + with open(os.path.join(self.graph_results_dir, filename), 'r') as fp: + expected_results = json.load(fp) + assert result == expected_results + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("node_name, result_file", [ + ('Default/bn1/Reshape1', "test_query_single_node_success_result1.json") + ]) + def test_query_single_node_success(self, client, node_name, result_file): + """Query single node.""" + train_id = gbl.get_train_ids()[0] + + params = dict(train_id=train_id, + name=node_name) + url = get_url(BASE_URL, params) + response = client.get(url) + assert response.status_code == 200 + self.compare_result_with_file(response.get_json(), result_file) diff --git a/tests/st/func/datavisual/graph/test_search_nodes_restful_api.py b/tests/st/func/datavisual/graph/test_search_nodes_restful_api.py new file mode 100644 index 00000000..fa848797 --- /dev/null +++ b/tests/st/func/datavisual/graph/test_search_nodes_restful_api.py @@ -0,0 +1,61 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test search nodes restful api. +Usage: + pytest tests/st/func/datavisual +""" +import os +import json + +import pytest +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +BASE_URL = '/v1/mindinsight/datavisual/graphs/nodes/names' + +class TestSearchNodes: + """Test search nodes restful APIs.""" + + graph_results_dir = os.path.join(os.path.dirname(__file__), 'graph_results') + + def compare_result_with_file(self, result, filename): + """Compare result with file which contain the expected results.""" + with open(os.path.join(self.graph_results_dir, filename), 'r') as fp: + expected_results = json.load(fp) + assert result == expected_results + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("search_content, offset, limit, result_file", [ + ('Default/bn1/Reshape', 0, 1000, "test_search_nodes_success_result1.json") + ]) + def test_search_nodes_success(self, client, search_content, offset, limit, result_file): + """Search node with parameters: offset is 0, limit is 1000.""" + train_id = gbl.get_train_ids()[0] + params = dict(train_id=train_id, + search=search_content, + offset=offset, + limit=limit) + url = get_url(BASE_URL, params) + response = client.get(url) + assert response.status_code == 200 + self.compare_result_with_file(response.get_json(), result_file) diff --git a/tests/st/func/datavisual/image/__init__.py b/tests/st/func/datavisual/image/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/image/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/image/test_metadata_restful_api.py b/tests/st/func/datavisual/image/test_metadata_restful_api.py new file mode 100644 index 00000000..84546cba --- /dev/null +++ b/tests/st/func/datavisual/image/test_metadata_restful_api.py @@ -0,0 +1,94 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Functions: + Test image metadata restful api. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.constants import MULTIPLE_TRAIN_ID, RESERVOIR_TRAIN_ID +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +from mindinsight.conf import settings +from mindinsight.datavisual.common.enums import PluginNameEnum + +BASE_URL = '/v1/mindinsight/datavisual/image/metadata' + + +class TestMetadata: + """Test Metadata.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_metadata(self, client): + """Test getting image metadata.""" + plugin_name = PluginNameEnum.IMAGE.value + train_id = gbl.get_train_ids()[0] + tag_name = gbl.get_tags(train_id, plugin_name)[0] + expected_metadata = gbl.get_metadata(train_id, tag_name) + + params = dict(train_id=train_id, tag=tag_name) + url = get_url(BASE_URL, params) + response = client.get(url) + metadata = response.get_json().get("metadatas") + + assert metadata == expected_metadata + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_metadata_multiple(self, client): + """Test getting image metadata, mainly test a summary dir with multiple logs.""" + plugin_name = PluginNameEnum.IMAGE.value + train_id = MULTIPLE_TRAIN_ID + tag_name = gbl.get_tags(train_id, plugin_name)[0] + expected_metadata = gbl.get_metadata(train_id, tag_name) + + params = dict(train_id=train_id, tag=tag_name) + url = get_url(BASE_URL, params) + response = client.get(url) + metadata = response.get_json().get("metadatas") + + assert metadata == expected_metadata + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_metadata_reservoir(self, client): + """Test getting image metadata, mainly test reservoir.""" + plugin_name = PluginNameEnum.IMAGE.value + train_id = RESERVOIR_TRAIN_ID + tag_name = gbl.get_tags(train_id, plugin_name)[0] + params = dict(train_id=train_id, tag=tag_name) + url = get_url(BASE_URL, params) + response = client.get(url) + metadata = response.get_json().get("metadatas") + assert len(metadata) == settings.MAX_IMAGE_STEP_SIZE_PER_TAG diff --git a/tests/st/func/datavisual/image/test_single_image_restful_api.py b/tests/st/func/datavisual/image/test_single_image_restful_api.py new file mode 100644 index 00000000..b84e7a61 --- /dev/null +++ b/tests/st/func/datavisual/image/test_single_image_restful_api.py @@ -0,0 +1,195 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Functions: + Test image single image restful api. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url, get_image_tensor_from_bytes + +from mindinsight.datavisual.common.enums import PluginNameEnum + +BASE_URL = '/v1/mindinsight/datavisual/image/single-image' + + +class TestSingleImage: + """Test query single image.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image(self, client): + """Test getting single image.""" + step = 1 + plugin_name = PluginNameEnum.IMAGE.value + train_id = gbl.get_train_ids()[0] + tag_name = gbl.get_tags(train_id, plugin_name)[0] + expected_image_tensor = gbl.get_single_image(train_id, tag_name, step) + + params = dict(train_id=train_id, tag=tag_name, step=step) + url = get_url(BASE_URL, params) + response = client.get(url) + recv_image_tensor = get_image_tensor_from_bytes(response.data) + + assert expected_image_tensor.any() == recv_image_tensor.any() + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image_no_train_id(self, client): + """Test getting single image without train id.""" + params = dict(tag="tag_name_0/image", step=1) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'train_id' is required." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image_no_tag(self, client): + """Test getting single image without tag.""" + params = dict(train_id="./summary0", step=1) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'tag' is required." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image_no_step(self, client): + """Test getting single image without step.""" + params = dict(train_id="./summary0", tag="tag_name_0/image") + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'step' is required." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("train_id", ["@#$", "./summary_x", dict()]) + def test_single_image_with_special_train_id(self, client, train_id): + """Test passing train_id with special character, invalid value, and wrong type.""" + params = dict(train_id=train_id, tag="tag_name_0/image", step=1) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. Can not find any data " \ + "in loader pool about the train job." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("tag", ["@#$", "tag_name_xxx/image", dict()]) + def test_single_image_with_special_tag(self, client, tag): + """Test passing tag with special character, invalid value, and wrong type.""" + train_id = gbl.get_train_ids()[0] + params = dict(train_id=train_id, tag=tag, step=1) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. Can not find any data " \ + "in this train job by given tag." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image_with_invalid_step(self, client): + """Test getting single image with invalid step.""" + train_id = gbl.get_train_ids()[0] + params = dict(train_id=train_id, tag="tag_name_0/image", step=1000) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. Can not find the step " \ + "with given train job id and tag." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("step", ["@#$", dict()]) + def test_single_image_with_special_step(self, client, step): + """Test getting single image with special step.""" + params = dict(train_id="./summary0", tag="tag_name_0/image", step=step) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540001' + assert response['error_msg'] == "Invalid parameter type. 'step' expect Integer type." diff --git a/tests/st/func/datavisual/scalar/__init__.py b/tests/st/func/datavisual/scalar/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/scalar/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/scalar/test_metadata_restful_api.py b/tests/st/func/datavisual/scalar/test_metadata_restful_api.py new file mode 100644 index 00000000..e3686d77 --- /dev/null +++ b/tests/st/func/datavisual/scalar/test_metadata_restful_api.py @@ -0,0 +1,55 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test scalar metadata restful api. +Usage: + pytest tests/st/func/datavisual +""" +import pytest +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +from mindinsight.datavisual.common.enums import PluginNameEnum + +BASE_URL = '/v1/mindinsight/datavisual/scalar/metadata' + + +class TestMetadata: + """Test Metadata.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_metadata(self, client): + """Test getting scalar metadata.""" + plugin_name = PluginNameEnum.SCALAR.value + train_id = gbl.get_train_ids()[0] + tag_name = gbl.get_tags(train_id, plugin_name)[0] + expected_metadata = gbl.get_metadata(train_id, tag_name) + + params = dict(train_id=train_id, tag=tag_name) + url = get_url(BASE_URL, params) + response = client.get(url) + metadata = response.get_json().get("metadatas") + + for metadata, expected_metadata in zip(metadata, expected_metadata): + assert metadata.get("wall_time") == expected_metadata.get("wall_time") + assert metadata.get("step") == expected_metadata.get("step") + assert metadata.get("value") - expected_metadata.get("value") < 1e-6 diff --git a/tests/st/func/datavisual/taskmanager/__init__.py b/tests/st/func/datavisual/taskmanager/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/taskmanager/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/taskmanager/test_plugins_restful_api.py b/tests/st/func/datavisual/taskmanager/test_plugins_restful_api.py new file mode 100644 index 00000000..59e717c4 --- /dev/null +++ b/tests/st/func/datavisual/taskmanager/test_plugins_restful_api.py @@ -0,0 +1,135 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test plugins restful api. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +from mindinsight.datavisual.common.enums import PluginNameEnum + +BASE_URL = '/v1/mindinsight/datavisual/plugins' + + +class TestPlugins: + """Test Plugins.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_plugins(self, client): + """Test getting plugins.""" + train_id = gbl.get_train_ids()[0] + expected_plugins = gbl.summaries_metadata.get(train_id).get("plugins") + + params = dict(train_id=train_id) + url = get_url(BASE_URL, params) + response = client.get(url) + plugins = response.get_json().get('plugins') + for plugin_name in PluginNameEnum.list_members(): + if plugin_name == PluginNameEnum.GRAPH.value: + assert len(plugins.get(plugin_name)) == len(expected_plugins.get(plugin_name)) + else: + assert sorted(plugins.get(plugin_name)) == sorted(expected_plugins.get(plugin_name)) + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_plugins_no_train_id(self, client): + """Test getting plugins without train_id.""" + params = dict() + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'train_id' is required." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("train_id", ["@#$", "./\x00home", "././/not_exist_id", dict()]) + def test_plugins_with_special_train_id(self, client, train_id): + """Test passing train_id with special character, null_byte, invalid id, and wrong type.""" + params = dict(train_id=train_id) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. Can not find " \ + "the train job in data manager." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + @pytest.mark.parametrize("manual_update", ["@#$", "tur", dict()]) + def test_plugins_with_special_manual_update(self, client, manual_update): + """Test passing manual_update with special character, wrong value, and wrong type.""" + train_id = gbl.get_train_ids()[0] + params = dict(train_id=train_id, manual_update=manual_update) + url = get_url(BASE_URL, params) + + response = client.get(url) + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. The value of " \ + "manual_update must be 'false' or 'true'." + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_plugins_with_train_id_not_in_cache(self, client): + """Test getting plugins with train id that not in loader pool.""" + train_id = "./summary0" + params = dict(train_id=train_id) + url = get_url(BASE_URL, params) + response = client.get(url) + plugins = response.get_json().get('plugins') + + for plugin_name in PluginNameEnum.list_members(): + # Empty list. + assert not plugins.get(plugin_name) diff --git a/tests/st/func/datavisual/taskmanager/test_query_single_train_task_restful_api.py b/tests/st/func/datavisual/taskmanager/test_query_single_train_task_restful_api.py new file mode 100644 index 00000000..3146717e --- /dev/null +++ b/tests/st/func/datavisual/taskmanager/test_query_single_train_task_restful_api.py @@ -0,0 +1,53 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test query single train task module. +Usage: + pytest tests/st/func/datavisual +""" +import pytest +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +from mindinsight.datavisual.common.enums import PluginNameEnum + +BASE_URL = '/v1/mindinsight/datavisual/single-job' + + +class TestQuerySingleTrainTask: + """Test Query Single Train Task.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_single_train_task(self, client): + """"Test query single train task.""" + for train_id in gbl.summaries_metadata: + expected = gbl.summaries_metadata.get(train_id).get("plugins") + for plugin_name in PluginNameEnum.list_members(): + params = dict(train_id=train_id, plugin_name=plugin_name) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + tags = result["train_jobs"][0]["tags"] + if plugin_name == PluginNameEnum.GRAPH.value: + assert len(tags) == len(expected[plugin_name]) + else: + assert sorted(tags) == sorted(expected[plugin_name]) diff --git a/tests/st/func/datavisual/taskmanager/test_query_train_jobs_restful_api.py b/tests/st/func/datavisual/taskmanager/test_query_train_jobs_restful_api.py new file mode 100644 index 00000000..8c20621c --- /dev/null +++ b/tests/st/func/datavisual/taskmanager/test_query_train_jobs_restful_api.py @@ -0,0 +1,125 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + test query train jobs module. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.constants import SUMMARY_DIR_NUM +from tests.st.func.datavisual.utils.utils import get_url + +BASE_URL = '/v1/mindinsight/datavisual/train-jobs' + + +class TestQueryTrainJobs: + """Test Query Train Jobs.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs(self, client): + """Test query train jobs.""" + params = dict(offset=0, limit=10) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('total') == SUMMARY_DIR_NUM + assert len(result.get('train_jobs')) == min(10, SUMMARY_DIR_NUM) + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs_with_large_offset(self, client): + """Test query train jobs with large offset.""" + params = dict(offset=10000, limit=10) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('total') == SUMMARY_DIR_NUM + assert len(result.get('train_jobs')) == min( + max(0, SUMMARY_DIR_NUM - 1000), 10) + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs_with_exceeded_limit(self, client): + """Test query train jobs with exceeded limit.""" + params = dict(offset=0, limit=1000) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('error_code') == '50540002' + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs_without_offset(self, client): + """Test query train jobs without offset.""" + params = dict(limit=10) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('total') == SUMMARY_DIR_NUM + assert len(result.get('train_jobs')) == min(10, SUMMARY_DIR_NUM) + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs_with_wrong_offse(self, client): + """Test query train jobs with wrong offse.""" + params = dict(offse=0, limit=10) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('total') == SUMMARY_DIR_NUM + assert len(result.get('train_jobs')) == min(10, SUMMARY_DIR_NUM) + + @pytest.mark.level1 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_query_train_jobs_with_lower_offset(self, client): + """Test query train jobs with lower offset.""" + params = dict(offset=-1, limit=10) + url = get_url(BASE_URL, params) + response = client.get(url) + result = response.get_json() + assert result.get('error_code') == '50540002' diff --git a/tests/st/func/datavisual/utils/__init__.py b/tests/st/func/datavisual/utils/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/utils/crc32.py b/tests/st/func/datavisual/utils/crc32.py new file mode 100644 index 00000000..06c3a00d --- /dev/null +++ b/tests/st/func/datavisual/utils/crc32.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mask string to crc32.""" +CRC_TABLE_32 = ( + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, + 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, + 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, + 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, + 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, + 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, + 0x6EF07595, 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, 0x5125DAD3, + 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, + 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, 0x61C69362, 0x93AD1061, 0x80FDE395, + 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, + 0x44694011, 0x5739B3E5, 0xA55230E6, 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, + 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, + 0x563C5F93, 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, 0x1871A4D8, + 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5, + 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, + 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, + 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, + 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, + 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, + 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, + 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, 0xF36E6F75, 0x0105EC76, 0x12551F82, + 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +) + +_CRC = 0 +_MASK = 0xFFFFFFFF + + +def _uint32(x): + """Transform x's type to uint32.""" + return x & 0xFFFFFFFF + + +def _get_crc_checksum(crc, data): + """Get crc checksum.""" + crc ^= _MASK + for d in data: + crc_table_index = (crc ^ d) & 0xFF + crc = (CRC_TABLE_32[crc_table_index] ^ (crc >> 8)) & _MASK + + crc ^= _MASK + + return crc + + +def get_mask_from_string(data): + """ + Get masked crc from data. + + Args: + data (byte): Byte string of data. + + Returns: + uint32, masked crc. + + """ + + crc = _get_crc_checksum(_CRC, data) + crc = _uint32(crc & _MASK) + crc = _uint32(((crc >> 15) | _uint32(crc << 17)) + 0xA282EAD8) + + return crc diff --git a/tests/st/func/datavisual/utils/globals.py b/tests/st/func/datavisual/utils/globals.py new file mode 100644 index 00000000..ce2f1689 --- /dev/null +++ b/tests/st/func/datavisual/utils/globals.py @@ -0,0 +1,50 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Description: Global variables. +""" +summaries_metadata = dict() +mock_data_manager = None + + +def get_tags(train_id, plugin_name): + """ + Get plugins. + + Args: + train_id (str): The train ID. + plugin_name (str): The plugin name. + + Returns: + list, tag names by given `train_id` and `plugin_name`. + + """ + return summaries_metadata.get(train_id).get("plugins").get(plugin_name) + + +def get_single_image(train_id, tag_name, step): + """Get single image.""" + data = summaries_metadata.get(train_id).get("actual_values").get(tag_name).get(step) + return data + + +def get_metadata(train_id, tag_name): + """Get metadata.""" + return summaries_metadata.get(train_id).get("metadata").get(tag_name) + + +def get_train_ids(): + """Get train ids.""" + return list(summaries_metadata.keys()) diff --git a/tests/st/func/datavisual/utils/log_generators/__init__.py b/tests/st/func/datavisual/utils/log_generators/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/datavisual/utils/log_generators/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/utils/log_generators/graph_base.json b/tests/st/func/datavisual/utils/log_generators/graph_base.json new file mode 100644 index 00000000..ff4d5416 --- /dev/null +++ b/tests/st/func/datavisual/utils/log_generators/graph_base.json @@ -0,0 +1,1414 @@ +{ + "node": [ + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + }, + { + "name": "x1", + "type": "DATA_EDGE" + }, + { + "name": "x2", + "type": "DATA_EDGE" + }, + { + "name": "x3", + "type": "DATA_EDGE" + }, + { + "name": "x4", + "type": "DATA_EDGE" + }, + { + "name": "x5", + "type": "DATA_EDGE" + }, + { + "name": "x6", + "type": "DATA_EDGE" + }, + { + "name": "x7", + "type": "DATA_EDGE" + }, + { + "name": "x8", + "type": "DATA_EDGE" + }, + { + "name": "x9", + "type": "DATA_EDGE" + }, + { + "name": "x10", + "type": "DATA_EDGE" + }, + { + "name": "conv1.weight", + "type": "DATA_EDGE" + } + ], + "name": "1", + "opType": "Conv2D", + "scope": "Default/conv1-Conv2d", + "attribute": [ + { + "name": "output_names", + "value": { + "dtype": "DT_GRAPHS", + "values": [ + { + "dtype": "DT_FLOAT64", + "strVal": "output" + } + ] + } + }, + { + "name": "pad_mode", + "value": { + "dtype": "DT_FLOAT64", + "strVal": "same" + } + } + ], + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "64" + }, + { + "size": "112" + }, + { + "size": "112" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + }, + { + "name": "x1", + "type": "DATA_EDGE" + }, + { + "name": "x2", + "type": "DATA_EDGE" + }, + { + "name": "x3", + "type": "DATA_EDGE" + }, + { + "name": "x4", + "type": "DATA_EDGE" + }, + { + "name": "x5", + "type": "DATA_EDGE" + }, + { + "name": "x6", + "type": "DATA_EDGE" + }, + { + "name": "x7", + "type": "DATA_EDGE" + }, + { + "name": "x8", + "type": "DATA_EDGE" + }, + { + "name": "x9", + "type": "DATA_EDGE" + }, + { + "name": "x10", + "type": "DATA_EDGE" + }, + { + "name": "cst13", + "type": "DATA_EDGE" + } + ], + "name": "53", + "opType": "tuple_getitem", + "scope": "Default/bn1-BatchNorm2d", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + }, + { + "name": "x12", + "type": "DATA_EDGE" + }, + { + "name": "x13", + "type": "DATA_EDGE" + }, + { + "name": "x14", + "type": "DATA_EDGE" + }, + { + "name": "x15", + "type": "DATA_EDGE" + }, + { + "name": "x16", + "type": "DATA_EDGE" + }, + { + "name": "x17", + "type": "DATA_EDGE" + }, + { + "name": "x18", + "type": "DATA_EDGE" + }, + { + "name": "x19", + "type": "DATA_EDGE" + }, + { + "name": "x20", + "type": "DATA_EDGE" + }, + { + "name": "conv1.weight", + "type": "DATA_EDGE" + }, + { + "name": "cst25", + "type": "DATA_EDGE" + } + ], + "name": "105", + "opType": "tuple_getitem", + "scope": "Default/bn1-BatchNorm2d", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "50", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "51", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "52", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "53", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "54", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "50", + "type": "DATA_EDGE" + } + ], + "name": "1", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "51", + "type": "DATA_EDGE" + } + ], + "name": "2", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "52", + "type": "DATA_EDGE" + } + ], + "name": "3", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "53", + "type": "DATA_EDGE" + } + ], + "name": "4", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "54", + "type": "DATA_EDGE" + } + ], + "name": "5", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "6", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "7", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "8", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "9", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "10", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "11", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "12", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + } + ], + "name": "849_848_847_424_1_construct", + "parameters": [ + { + "name": "x", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "conv1.weight", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "64" + }, + { + "size": "3" + }, + { + "size": "7" + }, + { + "size": "7" + } + ] + } + } + } + }, + { + "name": "x1", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x2", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x3", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x4", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x5", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x6", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x7", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x8", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x9", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x10", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x11", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x12", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x13", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x14", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x15", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x16", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x17", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x18", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x19", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x20", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + } + ], + "outputs": [ + { + "name": "228", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "10" + } + ] + } + } + } + } + ], + "constVals": [ + { + "key": "cst12", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + }, + { + "key": "cst13", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + }, + { + "key": "cst25", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + } + ] +} diff --git a/tests/st/func/datavisual/utils/log_generators/graph_log_generator.py b/tests/st/func/datavisual/utils/log_generators/graph_log_generator.py new file mode 100644 index 00000000..0afb0a81 --- /dev/null +++ b/tests/st/func/datavisual/utils/log_generators/graph_log_generator.py @@ -0,0 +1,79 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log generator for graph.""" +import json +import os +import time + +from google.protobuf import json_format +from tests.st.func.datavisual.utils.log_generators.log_generator import LogGenerator + +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 + + +class GraphLogGenerator(LogGenerator): + """ + Log generator for graph. + + This is a log generator writing graph. User can use it to generate fake + summary logs about graph. + """ + + def generate_log(self, file_path, graph_dict): + """ + Generate log for external calls. + + Args: + file_path (str): Path to write logs. + graph_dict (dict): A dict consists of graph node information. + + Returns: + dict, generated scalar metadata. + + """ + + graph_event = self.generate_event(dict(graph=graph_dict)) + + self._write_log_from_event(file_path, graph_event) + + return graph_dict + + def generate_event(self, values): + """ + Method for generating graph event. + + Args: + values (dict): Graph values. e.g. {'graph': graph_dict}. + + Returns: + summary_pb2.Event. + + """ + graph_json = { + 'wall_time': time.time(), + 'graph_def': values.get('graph'), + } + graph_event = json_format.Parse(json.dumps(graph_json), summary_pb2.Event()) + + return graph_event + + +if __name__ == "__main__": + graph_log_generator = GraphLogGenerator() + test_file_name = '%s.%s.%s' % ('graph', 'summary', str(time.time())) + graph_base_path = os.path.join(os.path.dirname(__file__), os.pardir, "log_generators", "graph_base.json") + with open(graph_base_path, 'r') as load_f: + graph = json.load(load_f) + graph_log_generator.generate_log(test_file_name, graph) diff --git a/tests/st/func/datavisual/utils/log_generators/images_log_generator.py b/tests/st/func/datavisual/utils/log_generators/images_log_generator.py new file mode 100644 index 00000000..02307506 --- /dev/null +++ b/tests/st/func/datavisual/utils/log_generators/images_log_generator.py @@ -0,0 +1,158 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log generator for images.""" +import io +import time + +import numpy as np +from PIL import Image +from tests.st.func.datavisual.utils.log_generators.log_generator import LogGenerator + +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 + + +class ImagesLogGenerator(LogGenerator): + """ + Log generator for images. + + This is a log generator writing images. User can use it to generate fake + summary logs about images. + """ + + def generate_event(self, values): + """ + Method for generating image event. + + Args: + values (dict): A dict contains: + { + wall_time (float): Timestamp. + step (int): Train step. + image (np.array): Pixels tensor. + tag (str): Tag name. + } + + Returns: + summary_pb2.Event. + + """ + + image_event = summary_pb2.Event() + image_event.wall_time = values.get('wall_time') + image_event.step = values.get('step') + + height, width, channel, image_string = self._get_image_string(values.get('image')) + value = image_event.summary.value.add() + value.tag = values.get('tag') + value.image.height = height + value.image.width = width + value.image.colorspace = channel + value.image.encoded_image = image_string + + return image_event + + def _get_image_string(self, image_tensor): + """ + Generate image string from tensor. + + Args: + image_tensor (np.array): Pixels tensor. + + Returns: + int, height. + int, width. + int, channel. + bytes, image_string. + + """ + height, width, channel = image_tensor.shape + scaled_height = int(height) + scaled_width = int(width) + + image = Image.fromarray(image_tensor) + image = image.resize((scaled_width, scaled_height), Image.ANTIALIAS) + output = io.BytesIO() + image.save(output, format='PNG') + image_string = output.getvalue() + output.close() + return height, width, channel, image_string + + def _make_image_tensor(self, shape): + """ + Make image tensor according to shape. + + Args: + shape (list): Shape of image, consists of height, width, channel. + + Returns: + np.array, image tensor. + + """ + image = np.prod(shape) + image_tensor = (np.arange(image, dtype=float)).reshape(shape) + image_tensor = image_tensor / np.max(image_tensor) * 255 + image_tensor = image_tensor.astype(np.uint8) + + return image_tensor + + def generate_log(self, file_path, steps_list, tag_name): + """ + Generate log for external calls. + + Args: + file_path (str): Path to write logs. + steps_list (list): A list consists of step. + tag_name (str): Tag name. + + Returns: + list[dict], generated image metadata. + dict, generated image tensors. + + """ + + images_values = dict() + images_metadata = [] + for step in steps_list: + wall_time = time.time() + + # height, width, channel + image_tensor = self._make_image_tensor([5, 5, 3]) + + image_metadata = dict() + image_metadata.update({'wall_time': wall_time}) + image_metadata.update({'step': step}) + image_metadata.update({'height': image_tensor.shape[0]}) + image_metadata.update({'width': image_tensor.shape[1]}) + images_metadata.append(image_metadata) + images_values.update({step: image_tensor}) + + values = dict( + wall_time=wall_time, + step=step, + image=image_tensor, + tag=tag_name + ) + + self._write_log_one_step(file_path, values) + + return images_metadata, images_values + + +if __name__ == "__main__": + images_log_generator = ImagesLogGenerator() + test_file_name = '%s.%s.%s' % ('image', 'summary', str(time.time())) + test_steps = [1, 3, 5] + test_tags = "test_image_tag_name" + images_log_generator.generate_log(test_file_name, test_steps, test_tags) diff --git a/tests/st/func/datavisual/utils/log_generators/log_generator.py b/tests/st/func/datavisual/utils/log_generators/log_generator.py new file mode 100644 index 00000000..f5a92228 --- /dev/null +++ b/tests/st/func/datavisual/utils/log_generators/log_generator.py @@ -0,0 +1,76 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base class for log generators.""" + +import struct +from abc import abstractmethod + +from tests.st.func.datavisual.utils import crc32 + + +class LogGenerator: + """ + Base log generator. + + This is a base class for log generators. User can use it to generate fake + summary logs. + """ + + @abstractmethod + def generate_event(self, values): + """ + Abstract method for generating event. + + Args: + values (dict): Values. + + Returns: + summary_pb2.Event. + + """ + + def _write_log_one_step(self, file_path, values): + """ + Write log one step. + + Args: + file_path (str): File path to write. + values (dict): Values. + + """ + event = self.generate_event(values) + self._write_log_from_event(file_path, event) + + @staticmethod + def _write_log_from_event(file_path, event): + """ + Write log by event. + + Args: + file_path (str): File path to write. + event (summary_pb2.Event): Event object in proto. + + """ + send_msg = event.SerializeToString() + + header = struct.pack(' 0: + time.sleep(first_sleep_time) + start_time = time.time() + while data_manager.status != DataManagerStatus.DONE.value: + time_used = time.time() - start_time + if time_used > time_limit: + break + time.sleep(0.1) + continue + + +def get_image_tensor_from_bytes(image_string): + """Get image tensor from bytes.""" + img = Image.open(io.BytesIO(image_string)) + image_tensor = np.array(img) + + return image_tensor diff --git a/tests/st/func/datavisual/workflow/__init__.py b/tests/st/func/datavisual/workflow/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/st/func/datavisual/workflow/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/datavisual/workflow/test_image_metadata.py b/tests/st/func/datavisual/workflow/test_image_metadata.py new file mode 100644 index 00000000..23ec115a --- /dev/null +++ b/tests/st/func/datavisual/workflow/test_image_metadata.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test workflow to query image metadata. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url + +from mindinsight.datavisual.common.enums import PluginNameEnum + +TRAIN_JOB_URL = '/v1/mindinsight/datavisual/train-jobs' +PLUGIN_URL = '/v1/mindinsight/datavisual/plugins' +METADATA_URL = '/v1/mindinsight/datavisual/image/metadata' + + +class TestImageMetadataFlow: + """Test Image Metadata.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_image_metadata(self, client): + """Test getting image metadata.""" + plugin_name = PluginNameEnum.IMAGE.value + + # Get train id by restful api `train-jobs`. + response = client.get(get_url(TRAIN_JOB_URL, dict())) + train_jobs = response.get_json() + train_id = train_jobs.get('train_jobs')[-1].get('train_id') + + # Get tag by restful api `plugins`. + params = dict(train_id=train_id, plugin=plugin_name) + response = client.get(get_url(PLUGIN_URL, params)) + plugins = response.get_json().get('plugins') + test_image_tag = plugins.get(plugin_name)[0] + + expected_metadata = gbl.get_metadata(train_id, test_image_tag) + + # Query image metadata. + params = dict(train_id=train_id, tag=test_image_tag) + response = client.get(get_url(METADATA_URL, params)) + metadata = response.get_json().get("metadatas") + + assert metadata == expected_metadata diff --git a/tests/st/func/datavisual/workflow/test_single_image.py b/tests/st/func/datavisual/workflow/test_single_image.py new file mode 100644 index 00000000..6fa16b17 --- /dev/null +++ b/tests/st/func/datavisual/workflow/test_single_image.py @@ -0,0 +1,73 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test workflow to query single image. +Usage: + pytest tests/st/func/datavisual +""" +import pytest + +from tests.st.func.datavisual.utils import globals as gbl +from tests.st.func.datavisual.utils.utils import get_url, get_image_tensor_from_bytes + +from mindinsight.datavisual.common.enums import PluginNameEnum + +TRAIN_JOB_URL = '/v1/mindinsight/datavisual/train-jobs' +PLUGIN_URL = '/v1/mindinsight/datavisual/plugins' +METADATA_URL = '/v1/mindinsight/datavisual/image/metadata' +SINGLE_IMAGE_URL = '/v1/mindinsight/datavisual/image/single-image' + + +class TestSingleImageFlow: + """Test Single Image.""" + + @pytest.mark.level0 + @pytest.mark.env_single + @pytest.mark.platform_x86_cpu + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.usefixtures("init_summary_logs") + def test_single_image(self, client): + """Test getting single image.""" + plugin_name = PluginNameEnum.IMAGE.value + + # Get train id by restful api `train-jobs`. + response = client.get(get_url(TRAIN_JOB_URL, dict())) + train_jobs = response.get_json() + train_id = train_jobs.get('train_jobs')[0].get('train_id') + + # Get tag by restful api `plugins`. + params = dict(train_id=train_id, plugin=plugin_name) + response = client.get(get_url(PLUGIN_URL, params)) + plugins = response.get_json().get('plugins') + test_image_tag = plugins.get(plugin_name)[0] + + # Get step by restful api `metadata`. + params = dict(train_id=train_id, tag=test_image_tag) + response = client.get(get_url(METADATA_URL, params)) + metadata = response.get_json().get("metadatas") + test_step = metadata[0].get('step') + + # Query single image. + expected_image_tensor = gbl.get_single_image(train_id, test_image_tag, test_step) + + params = dict(train_id=train_id, tag=test_image_tag, step=test_step) + url = get_url(SINGLE_IMAGE_URL, params) + response = client.get(url) + recv_image_tensor = get_image_tensor_from_bytes(response.data) + + assert expected_image_tensor.any() == recv_image_tensor.any() diff --git a/tests/st/func/lineagemgr/__init__.py b/tests/st/func/lineagemgr/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/api/__init__.py b/tests/st/func/lineagemgr/api/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/api/test_model_api.py b/tests/st/func/lineagemgr/api/test_model_api.py new file mode 100644 index 00000000..2adf15c8 --- /dev/null +++ b/tests/st/func/lineagemgr/api/test_model_api.py @@ -0,0 +1,843 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Function: + Test the query module about lineage information. +Usage: + The query module test should be run after lineagemgr/collection/model/test_model_lineage.py + pytest lineagemgr +""" + +import os +from unittest import TestCase + +import pytest + +from mindinsight.lineagemgr import get_summary_lineage, filter_summary_lineage +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageParamSummaryPathError, LineageParamValueError, LineageParamTypeError, \ + LineageSearchConditionParamError, LineageFileNotFoundError +from ..conftest import BASE_SUMMARY_DIR, SUMMARY_DIR, SUMMARY_DIR_2, DATASET_GRAPH + + +@pytest.mark.usefixtures("create_summary_dir") +class TestModelApi(TestCase): + """Test lineage information query interface.""" + + @classmethod + def setup_class(cls): + """The initial process.""" + cls.dir_with_empty_lineage = os.path.join( + BASE_SUMMARY_DIR, 'dir_with_empty_lineage' + ) + os.makedirs(cls.dir_with_empty_lineage) + ms_file = os.path.join( + cls.dir_with_empty_lineage, 'empty.summary.1581499502.bms_MS' + ) + lineage_file = ms_file + '_lineage' + with open(ms_file, mode='w'): + pass + with open(lineage_file, mode='w'): + pass + + cls.empty_dir = os.path.join(BASE_SUMMARY_DIR, 'empty_dir') + os.makedirs(cls.empty_dir) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_get_summary_lineage(self): + """Test the interface of get_summary_lineage.""" + total_res = get_summary_lineage(SUMMARY_DIR) + partial_res1 = get_summary_lineage(SUMMARY_DIR, ['hyper_parameters']) + partial_res2 = get_summary_lineage(SUMMARY_DIR, ['metric', 'algorithm']) + expect_total_res = { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'metric': { + 'accuracy': 0.78 + }, + 'hyper_parameters': { + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'epoch': 14, + 'parallel_mode': 'stand_alone', + 'device_num': 2, + 'batch_size': 32 + }, + 'algorithm': { + 'network': 'ResNet' + }, + 'train_dataset': { + 'train_dataset_size': 731 + }, + 'valid_dataset': { + 'valid_dataset_size': 10240 + }, + 'model': { + 'path': '{"ckpt": "' + + BASE_SUMMARY_DIR + '/run1/CKPtest_model.ckpt"}', + 'size': 64 + }, + 'dataset_graph': DATASET_GRAPH + } + expect_partial_res1 = { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'hyper_parameters': { + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'epoch': 14, + 'parallel_mode': 'stand_alone', + 'device_num': 2, + 'batch_size': 32 + } + } + expect_partial_res2 = { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'metric': { + 'accuracy': 0.78 + }, + 'algorithm': { + 'network': 'ResNet' + } + } + assert expect_total_res == total_res + assert expect_partial_res1 == partial_res1 + assert expect_partial_res2 == partial_res2 + + # the lineage summary file is empty + result = get_summary_lineage(self.dir_with_empty_lineage) + assert {} == result + + # keys is empty list + expect_result = { + 'summary_dir': SUMMARY_DIR + } + result = get_summary_lineage(SUMMARY_DIR, []) + assert expect_result == result + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_get_summary_lineage_exception(self): + """Test the interface of get_summary_lineage with exception.""" + # summary path does not exist + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path does not exist or is not a dir.', + get_summary_lineage, + '/tmp/fake/dir' + ) + + # summary path is relative path + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + get_summary_lineage, + 'tmp' + ) + + # the type of input param is invalid + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + get_summary_lineage, + ['/root/linage1', '/root/lineage2'] + ) + + # summary path is empty str + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + get_summary_lineage, + '', + keys=None + ) + + # summary path invalid + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + get_summary_lineage, + '\\', + keys=None + ) + + # keys is invalid + self.assertRaisesRegex( + LineageParamValueError, + 'Keys must be in', + get_summary_lineage, + SUMMARY_DIR, + ['metric', 'fake_name'] + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Keys must be list.', + get_summary_lineage, + SUMMARY_DIR, + 0 + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Keys must be list.', + get_summary_lineage, + SUMMARY_DIR, + 0.1 + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Keys must be list.', + get_summary_lineage, + SUMMARY_DIR, + True + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Element of keys must be str.', + get_summary_lineage, + SUMMARY_DIR, + [1, 2, 3] + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Keys must be list.', + get_summary_lineage, + SUMMARY_DIR, + (3, 4) + ) + + self.assertRaisesRegex( + LineageParamTypeError, + 'Keys must be list.', + get_summary_lineage, + SUMMARY_DIR, + {'a': 'b'} + ) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage(self): + """Test the interface of filter_summary_lineage.""" + expect_result = { + 'object': [ + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 1024, + 'test_dataset_path': None, + 'test_dataset_count': None, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 10, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 64, + 'metric': {}, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + }, + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 731, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': None, + 'model_size': 64, + 'metric': { + 'accuracy': 0.78 + }, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + }, + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), + 'loss_function': None, + 'train_dataset_path': None, + 'train_dataset_count': None, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': None, + 'optimizer': None, + 'learning_rate': None, + 'epoch': None, + 'batch_size': None, + 'loss': None, + 'model_size': None, + 'metric': { + 'accuracy': 2.7800000000000002 + }, + 'dataset_graph': {}, + 'dataset_mark': 3 + } + ], + 'count': 3 + } + + search_condition = { + 'sorted_name': 'summary_dir' + } + res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(res.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == res + + expect_result = { + 'object': [], + 'count': 0 + } + res = filter_summary_lineage(self.dir_with_empty_lineage) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(res.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == res + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_with_condition_1(self): + """Test the interface of filter_summary_lineage with condition.""" + search_condition = { + 'summary_dir': { + 'in': [ + SUMMARY_DIR, + SUMMARY_DIR_2 + ] + }, + 'metric_accuracy': { + 'lt': 3.0, + 'gt': 0.5 + }, + 'sorted_name': 'metric_accuracy', + 'sorted_type': 'descending', + 'limit': 3, + 'offset': 0 + } + expect_result = { + 'object': [ + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), + 'loss_function': None, + 'train_dataset_path': None, + 'train_dataset_count': None, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': None, + 'optimizer': None, + 'learning_rate': None, + 'epoch': None, + 'batch_size': None, + 'loss': None, + 'model_size': None, + 'metric': { + 'accuracy': 2.7800000000000002 + }, + 'dataset_graph': {}, + 'dataset_mark': 3 + }, + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 731, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': None, + 'model_size': 64, + 'metric': { + 'accuracy': 0.78 + }, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + } + ], + 'count': 2 + } + partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(partial_res.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == partial_res + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_with_condition_2(self): + """Test the interface of filter_summary_lineage with condition.""" + search_condition = { + 'summary_dir': { + 'in': [ + './run1', + './run2' + ] + }, + 'metric_accuracy': { + 'lt': 3.0, + 'gt': 0.5 + }, + 'sorted_name': 'metric_accuracy', + 'sorted_type': 'descending', + 'limit': 3, + 'offset': 0 + } + expect_result = { + 'object': [ + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), + 'loss_function': None, + 'train_dataset_path': None, + 'train_dataset_count': None, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': None, + 'optimizer': None, + 'learning_rate': None, + 'epoch': None, + 'batch_size': None, + 'loss': None, + 'model_size': None, + 'metric': { + 'accuracy': 2.7800000000000002 + }, + 'dataset_graph': {}, + 'dataset_mark': 3 + }, + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 731, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': None, + 'model_size': 64, + 'metric': { + 'accuracy': 0.78 + }, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + } + ], + 'count': 2 + } + partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(partial_res.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == partial_res + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_with_condition_3(self): + """Test the interface of filter_summary_lineage with condition.""" + search_condition1 = { + 'batch_size': { + 'ge': 30 + }, + 'sorted_name': 'metric_accuracy', + 'lineage_type': None + } + expect_result = { + 'object': [ + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 1024, + 'test_dataset_path': None, + 'test_dataset_count': None, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 10, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 64, + 'metric': {}, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + }, + { + 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 731, + 'test_dataset_path': None, + 'test_dataset_count': 10240, + 'network': 'ResNet', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': None, + 'model_size': 64, + 'metric': { + 'accuracy': 0.78 + }, + 'dataset_graph': DATASET_GRAPH, + 'dataset_mark': 2 + } + ], + 'count': 2 + } + partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(partial_res1.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == partial_res1 + + search_condition2 = { + 'batch_size': { + 'lt': 30 + }, + 'lineage_type': 'model' + } + expect_result = { + 'object': [], + 'count': 0 + } + partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) + expect_objects = expect_result.get('object') + for idx, res_object in enumerate(partial_res2.get('object')): + expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') + assert expect_result == partial_res2 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_with_lineage_type(self): + """Test the interface of filter_summary_lineage with lineage_type.""" + summary_dir = os.path.join(BASE_SUMMARY_DIR, 'except_run') + search_condition = { + 'summary_dir': { + 'in': [summary_dir] + }, + 'lineage_type': 'dataset' + } + expect_result = { + 'object': [ + { + 'summary_dir': summary_dir, + 'dataset_graph': DATASET_GRAPH + } + ], + 'count': 1 + } + res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) + assert expect_result == res + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_exception_1(self): + """Test the abnormal execution of the filter_summary_lineage interface.""" + # summary base dir is relative path + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + filter_summary_lineage, + 'relative_path' + ) + + # summary base dir does not exist + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path does not exist or is not a dir.', + filter_summary_lineage, + '/path/does/not/exist' + ) + + # no summary log file under summary_base_dir + self.assertRaisesRegex( + LineageFileNotFoundError, + 'There is no summary log file under summary_base_dir.', + filter_summary_lineage, + self.empty_dir + ) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_exception_2(self): + """Test the abnormal execution of the filter_summary_lineage interface.""" + # search_condition type error + search_condition = { + 'summary_dir': 1.0 + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The search_condition element summary_dir should be dict.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # only sorted_type (no sorted_name) + search_condition = { + 'sorted_type': 'descending' + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The sorted_name have to exist when sorted_type exists.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # search condition is not dict or None + search_condition = [] + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'Invalid search_condition type, it should be dict.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the condition of limit is invalid + search_condition = { + 'limit': True + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The limit must be int.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the condition of offset is invalid + search_condition = { + 'offset': 1.0 + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The offset must be int.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the search attribute not supported + search_condition = { + 'dataset_graph': { + 'le': 1 + } + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The search attribute not supported.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the sorted_name not supported + search_condition = { + 'sorted_name': 'xxx' + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The sorted_name must be in', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the sorted_type not supported + search_condition = { + 'sorted_name': 'summary_dir', + 'sorted_type': 'xxx' + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The sorted_type must be ascending or descending', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the search condition not supported + search_condition = { + 'batch_size': { + 'dd': 30 + } + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The compare condition should be in', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the search condition type error + search_condition = { + 'metric_accuracy': { + 'lt': 'xxx' + } + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The parameter metric_accuracy is invalid.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the summary dir is invalid in search condition + search_condition = { + 'summary_dir': { + 'in': [ + 'xxx' + ] + } + } + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + # the condition type not supported in summary dir + search_condition = { + 'summary_dir': { + 'lt': '/xxx' + } + } + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'Invalid operation of summary dir.', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + search_condition = { + 'sorted_name': 'metric_' + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The sorted_name must be in', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + search_condition = { + 'sorted_name': 1 + } + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'The sorted_name must be in', + filter_summary_lineage, + BASE_SUMMARY_DIR, + search_condition + ) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_filter_summary_lineage_exception_3(self): + """Test the abnormal execution of the filter_summary_lineage interface.""" + # gt > lt + search_condition1 = { + 'metric_accuracy': { + 'gt': 1, + 'lt': 0.5 + } + } + expect_result = { + 'object': [], + 'count': 0 + } + partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) + assert expect_result == partial_res1 + + # the (offset + 1) * limit > count + search_condition2 = { + 'loss': { + 'lt': 1 + }, + 'limit': 1, + 'offset': 4 + } + expect_result = { + 'object': [], + 'count': 1 + } + partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) + assert expect_result == partial_res2 diff --git a/tests/st/func/lineagemgr/collection/__init__.py b/tests/st/func/lineagemgr/collection/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/__init__.py b/tests/st/func/lineagemgr/collection/model/__init__.py new file mode 100644 index 00000000..47b43a6e --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/__init__.py new file mode 100644 index 00000000..b56125b7 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore Interface.""" +from .application.model_zoo.resnet import ResNet +from .common.tensor import Tensor +from .dataset import MindDataset, serialize +from .nn import * +from .train.callback import _ListCallback, Callback, RunContext, ModelCheckpoint, SummaryStep +from .train.summary import SummaryRecord diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/application/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/application/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/application/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py b/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py new file mode 100644 index 00000000..f9755ec9 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py @@ -0,0 +1,24 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore ResNet class.""" +from ...nn.cell import Cell + + +class ResNet(Cell): + """Mocked ResNet.""" + + def __init__(self): + super(ResNet, self).__init__() + self._cells = {} diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/common/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/common/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/common/tensor.py b/tests/st/func/lineagemgr/collection/model/mindspore/common/tensor.py new file mode 100644 index 00000000..33205845 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/common/tensor.py @@ -0,0 +1,30 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/common/tensor.py.""" +import numpy as np + + +class Tensor: + """Mock the MindSpore Tensor class.""" + + def __init__(self, value=0): + self._value = value + + def asnumpy(self): + """Get value in numpy format.""" + return np.array(self._value) + + def __repr__(self): + return str(self.asnumpy()) diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/context.py b/tests/st/func/lineagemgr/collection/model/mindspore/context.py new file mode 100644 index 00000000..e3dbef0e --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/context.py @@ -0,0 +1,21 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" + + +def get_context(key): + """Get key in context.""" + context = {"device_id": 1} + return context.get(key) diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/dataset/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/__init__.py new file mode 100644 index 00000000..a5061d00 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock mindspore.dataset.""" +from .engine import MindDataset, serialize diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py new file mode 100644 index 00000000..bc140176 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock mindspore.dataset.engine.""" +from .datasets import Dataset, MindDataset +from .serializer_deserializer import serialize diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py new file mode 100644 index 00000000..a9decbce --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py @@ -0,0 +1,36 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/dataset/engine/datasets.py.""" + + +class Dataset: + """Mock the MindSpore Dataset class.""" + + def __init__(self, dataset_size=None, dataset_path=None): + self.dataset_size = dataset_size + self.dataset_path = dataset_path + self.input = [] + + def get_dataset_size(self): + """Mocked get_dataset_size.""" + return self.dataset_size + + +class MindDataset(Dataset): + """Mock the MindSpore MindDataset class.""" + + def __init__(self, dataset_size=None, dataset_file=None): + super(MindDataset, self).__init__(dataset_size) + self.dataset_file = dataset_file diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/serializer_deserializer.py b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/serializer_deserializer.py new file mode 100644 index 00000000..da5e57bb --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/dataset/engine/serializer_deserializer.py @@ -0,0 +1,73 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/dataset/engine/serializer_deserializer.py.""" +import json + +SERIALIZED_PIPELINE = { + 'op_type': 'BatchDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'drop_remainder': True, + 'batch_size': 10, + 'children': [ + { + 'op_type': 'MapDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'input_columns': [ + 'label' + ], + 'output_columns': [ + None + ], + 'operations': [ + { + 'tensor_op_module': 'minddata.transforms.c_transforms', + 'tensor_op_name': 'OneHot', + 'num_classes': 10 + } + ], + 'children': [ + { + 'op_type': 'MnistDataset', + 'shard_id': None, + 'num_shards': None, + 'op_module': 'minddata.dataengine.datasets', + 'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData', + 'num_parallel_workers': None, + 'shuffle': None, + 'num_samples': 100, + 'sampler': { + 'sampler_module': 'minddata.dataengine.samplers', + 'sampler_name': 'RandomSampler', + 'replacement': True, + 'num_samples': 100 + }, + 'children': [] + } + ] + } + ] +} + +def serialize(dataset, json_filepath=None): + """Mock the MindSpore serialize method.""" + serialized_pipeline = SERIALIZED_PIPELINE + if dataset is None: + return json.dumps({'key': 'value'}) + if json_filepath: + with open(json_filepath, 'w') as json_file: + json.dump(serialized_pipeline, json_file, indent=2) + return serialized_pipeline diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/__init__.py new file mode 100644 index 00000000..4b8053a2 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the mindspore.nn package.""" +from .optim import Optimizer, Momentum +from .loss.loss import SoftmaxCrossEntropyWithLogits, _Loss +from .cell import Cell, WithLossCell, TrainOneStepWithLossScaleCell + +__all__ = ['Optimizer', 'Momentum', 'SoftmaxCrossEntropyWithLogits', + '_Loss', 'Cell', 'WithLossCell', + 'TrainOneStepWithLossScaleCell'] diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/cell.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/cell.py new file mode 100644 index 00000000..7a603e0d --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/cell.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/train/callback.py.""" +from collections import OrderedDict + + +class Cell: + """Mock the Cell class.""" + + def __init__(self, auto_prefix=True, pips=None): + if pips is None: + pips = dict() + self._cells = OrderedDict() + self._auto_prefix = auto_prefix + self._pips = pips + + @property + def auto_prefix(self): + """The property of auto_prefix.""" + return self._auto_prefix + + @property + def pips(self): + """The property of pips.""" + return self._pips + + +class WithLossCell(Cell): + """Mocked WithLossCell class.""" + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__() + self._backbone = backbone + self._loss_fn = loss_fn + + +class TrainOneStepWithLossScaleCell(Cell): + """Mocked TrainOneStepWithLossScaleCell.""" + def __init__(self, network, optimizer): + super(TrainOneStepWithLossScaleCell, self).__init__() + self.network = network + self.optimizer = optimizer diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/loss.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/loss.py new file mode 100644 index 00000000..7323a087 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/loss/loss.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore SoftmaxCrossEntropyWithLogits class.""" +from ..cell import Cell + + +class _Loss(Cell): + """Mocked _Loss.""" + + def __init__(self, reduction='mean'): + super(_Loss, self).__init__() + self.reduction = reduction + + def construct(self, base, target): + """Mocked construct function.""" + raise NotImplementedError + + +class SoftmaxCrossEntropyWithLogits(_Loss): + """Mocked SoftmaxCrossEntropyWithLogits.""" + + def __init__(self, weight=None): + super(SoftmaxCrossEntropyWithLogits, self).__init__(weight) + + def construct(self, base, target): + """Mocked construct.""" + return 1 diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/optim.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/optim.py new file mode 100644 index 00000000..84e9a8ab --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/optim.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/nn/optim.py.""" +from .cell import Cell + + +class Parameter: + """Mock the MindSpore Parameter class.""" + + def __init__(self, learning_rate): + self._name = "Parameter" + self.default_input = learning_rate + + @property + def name(self): + """The property of name.""" + return self._name + + def __repr__(self): + format_str = 'Parameter (name={name})' + return format_str.format(name=self._name) + + +class Optimizer(Cell): + """Mock the MindSpore Optimizer class.""" + + def __init__(self, learning_rate): + super(Optimizer, self).__init__() + self.learning_rate = Parameter(learning_rate) + + +class Momentum(Optimizer): + """Mock the MindSpore Momentum class.""" + + def __init__(self, learning_rate): + super(Momentum, self).__init__(learning_rate) + self.dynamic_lr = False diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py new file mode 100644 index 00000000..6d725bbe --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore wrap package.""" +from .loss_scale import TrainOneStepWithLossScaleCell +from .cell_wrapper import WithLossCell diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py new file mode 100644 index 00000000..49b84e42 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py @@ -0,0 +1,25 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore cell_wrapper.py.""" +from ..cell import Cell + + +class WithLossCell(Cell): + """Mock the WithLossCell class.""" + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__() + self._backbone = backbone + self._loss_fn = loss_fn diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py new file mode 100644 index 00000000..0473aacb --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py @@ -0,0 +1,29 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore loss_scale.py.""" +from ..cell import Cell + + +class TrainOneStepWithLossScaleCell(Cell): + """Mock the TrainOneStepWithLossScaleCell class.""" + + def __init__(self, network, optimizer): + super(TrainOneStepWithLossScaleCell, self).__init__() + self.network = network + self.optimizer = optimizer + + def construct(self, data, label): + """Mock the construct method.""" + raise NotImplementedError diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/train/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/train/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/train/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/train/callback.py b/tests/st/func/lineagemgr/collection/model/mindspore/train/callback.py new file mode 100644 index 00000000..7b1d39f2 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/train/callback.py @@ -0,0 +1,84 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/train/callback.py.""" +import os + + +class RunContext: + """Mock the RunContext class.""" + + def __init__(self, original_args=None): + self._original_args = original_args + self._stop_requested = False + + def original_args(self): + """Mock original_args.""" + return self._original_args + + def stop_requested(self): + """Mock stop_requested method.""" + return self._stop_requested + + +class Callback: + """Mock the Callback class.""" + + def __init__(self): + pass + + def begin(self, run_context): + """Called once before network training.""" + + def epoch_begin(self, run_context): + """Called before each epoch begin.""" + + +class _ListCallback(Callback): + """Mock the _ListCallabck class.""" + + def __init__(self, callbacks): + super(_ListCallback, self).__init__() + self._callbacks = callbacks + + +class ModelCheckpoint(Callback): + """Mock the ModelCheckpoint class.""" + + def __init__(self, prefix='CKP', directory=None, config=None): + super(ModelCheckpoint, self).__init__() + self._prefix = prefix + self._directory = directory + self._config = config + self._latest_ckpt_file_name = os.path.join(directory, prefix + 'test_model.ckpt') + + @property + def model_file_name(self): + """Get the file name of model.""" + return self._model_file_name + + @property + def latest_ckpt_file_name(self): + """Get the latest file name fo checkpoint.""" + return self._latest_ckpt_file_name + + +class SummaryStep(Callback): + """Mock the SummaryStep class.""" + + def __init__(self, summary, flush_step=10): + super(SummaryStep, self).__init__() + self._sumamry = summary + self._flush_step = flush_step + self.summary_file_name = summary.full_file_name diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/__init__.py b/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/__init__.py new file mode 100644 index 00000000..ed0eee75 --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" +from .summary_record import SummaryRecord + +__all__ = ["SummaryRecord"] diff --git a/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/summary_record.py b/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/summary_record.py new file mode 100644 index 00000000..4bf0f3dc --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/mindspore/train/summary/summary_record.py @@ -0,0 +1,44 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" +import os +import time + + +class SummaryRecord: + """Mock the MindSpore SummaryRecord class.""" + + def __init__(self, + log_dir: str, + file_prefix: str = "events.", + file_suffix: str = ".MS", + create_time=int(time.time())): + self.log_dir = log_dir + self.prefix = file_prefix + self.suffix = file_suffix + file_name = file_prefix + 'summary.' + str(create_time) + file_suffix + self.full_file_name = os.path.join(log_dir, file_name) + permissions = os.R_OK | os.W_OK | os.X_OK + mode = permissions << 6 + if not os.path.exists(log_dir): + os.makedirs(log_dir, mode=mode, exist_ok=True) + with open(self.full_file_name, 'wb'): + pass + + def flush(self): + """Mock flush method.""" + + def close(self): + """Mock close method.""" diff --git a/tests/st/func/lineagemgr/collection/model/test_model_lineage.py b/tests/st/func/lineagemgr/collection/model/test_model_lineage.py new file mode 100644 index 00000000..73f54dab --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/test_model_lineage.py @@ -0,0 +1,328 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +""" +Fuction: + Test the lineage information collection module. +Usage: + pytest lineagemgr/collection/model/test_model_lineage.py +""" + +import os +import shutil +import time +from unittest import mock, TestCase + +import numpy as np +import pytest + +from mindinsight.lineagemgr import get_summary_lineage +from mindinsight.lineagemgr.collection.model.model_lineage import TrainLineage, EvalLineage, \ + AnalyzeObject +from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ + LineageFileNotFoundError +from mindinsight.utils.exceptions import MindInsightException +from mindspore.application.model_zoo.resnet import ResNet +from mindspore.common.tensor import Tensor +from mindspore.dataset.engine import MindDataset +from mindspore.nn import Momentum, SoftmaxCrossEntropyWithLogits, WithLossCell +from mindspore.train.callback import RunContext, ModelCheckpoint, SummaryStep, _ListCallback +from mindspore.train.summary import SummaryRecord +from ...conftest import SUMMARY_DIR, SUMMARY_DIR_2, SUMMARY_DIR_3, BASE_SUMMARY_DIR +from .train_one_step import TrainOneStep + + +@pytest.mark.usefixtures("create_summary_dir") +class TestModelLineage(TestCase): + """Test mindinsight.lineagemgr.collection.model.model_lineage.py""" + + @classmethod + def setup_class(cls): + """Setup method.""" + cls.optimizer = Momentum(Tensor(0.12)) + cls.loss_fn = SoftmaxCrossEntropyWithLogits() + cls.net = ResNet() + + cls.run_context = dict() + cls.run_context['train_network'] = cls.net + cls.run_context['loss_fn'] = cls.loss_fn + cls.run_context['net_outputs'] = Tensor(np.array([0.03])) + cls.run_context['optimizer'] = cls.optimizer + cls.run_context['train_dataset'] = MindDataset(dataset_size=32) + cls.run_context['epoch_num'] = 10 + cls.run_context['cur_step_num'] = 320 + cls.run_context['parallel_mode'] = "stand_alone" + cls.run_context['device_number'] = 2 + cls.run_context['batch_num'] = 32 + cls.summary_record = SummaryRecord(SUMMARY_DIR) + callback = [ModelCheckpoint(directory=SUMMARY_DIR), + SummaryStep(cls.summary_record), + TrainLineage(cls.summary_record) + ] + cls.run_context['list_callback'] = _ListCallback(callback) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_train_begin(self): + """Test the begin function in TrainLineage.""" + train_callback = TrainLineage(self.summary_record, True) + train_callback.begin(RunContext(self.run_context)) + assert train_callback.initial_learning_rate == 0.12 + lineage_log_path = self.summary_record.full_file_name + '_lineage' + assert os.path.isfile(lineage_log_path) is True + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + @mock.patch.object(AnalyzeObject, 'get_file_size') + def test_training_end(self, *args): + """Test the end function in TrainLineage.""" + args[0].return_value = 64 + train_callback = TrainLineage(self.summary_record, True) + train_callback.initial_learning_rate = 0.12 + train_callback.end(RunContext(self.run_context)) + res = get_summary_lineage(SUMMARY_DIR) + assert res.get('hyper_parameters', {}).get('epoch') == 10 + run_context = self.run_context + run_context['epoch_num'] = 14 + train_callback.end(RunContext(run_context)) + res = get_summary_lineage(SUMMARY_DIR) + assert res.get('hyper_parameters', {}).get('epoch') == 14 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_eval_end(self): + """Test the end function in EvalLineage.""" + eval_callback = EvalLineage(self.summary_record, True) + eval_run_context = self.run_context + eval_run_context['metrics'] = {'accuracy': 0.78} + eval_run_context['valid_dataset'] = self.run_context['train_dataset'] + eval_run_context['step_num'] = 32 + eval_callback.end(RunContext(eval_run_context)) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_eval_only(self): + """Test record evaluation event only.""" + summary_dir = os.path.join(BASE_SUMMARY_DIR, 'eval_only_dir') + summary_record = SummaryRecord(summary_dir) + eval_run_context = self.run_context + eval_run_context['metrics'] = {'accuracy': 0.58} + eval_run_context['valid_dataset'] = self.run_context['train_dataset'] + eval_run_context['step_num'] = 32 + eval_only_callback = EvalLineage(summary_record) + eval_only_callback.end(RunContext(eval_run_context)) + res = get_summary_lineage(summary_dir, + ['metric', 'dataset_graph']) + expect_res = { + 'summary_dir': summary_dir, + 'dataset_graph': {}, + 'metric': {'accuracy': 0.58} + } + assert res == expect_res + shutil.rmtree(summary_dir) + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + @mock.patch('os.path.getsize') + def test_multiple_trains(self, *args): + """ + Callback TrainLineage and EvalLineage for mutltiple times. + + Write TrainLineage and EvalLineage in different files under same directory. + EvalLineage log file end with '_lineage'. + """ + args[0].return_value = 10 + for i in range(2): + summary_record = SummaryRecord(SUMMARY_DIR_2, + create_time=int(time.time()) + i) + eval_record = SummaryRecord(SUMMARY_DIR_2, + file_prefix='events.eval.', + create_time=int(time.time() + 10) + i, + file_suffix='.MS_lineage') + train_callback = TrainLineage(summary_record, True) + train_callback.begin(RunContext(self.run_context)) + train_callback.end(RunContext(self.run_context)) + eval_callback = EvalLineage(eval_record, True) + eval_run_context = self.run_context + eval_run_context['metrics'] = {'accuracy': 0.78 + i + 1} + eval_run_context['valid_dataset'] = self.run_context['train_dataset'] + eval_run_context['step_num'] = 32 + eval_callback.end(RunContext(eval_run_context)) + file_num = os.listdir(SUMMARY_DIR_2) + assert len(file_num) == 8 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + @mock.patch.object(AnalyzeObject, 'get_file_size') + def test_train_with_customized_network(self, *args): + """Test train with customized network.""" + args[0].return_value = 64 + train_callback = TrainLineage(self.summary_record, True) + run_context_customized = self.run_context + del run_context_customized['optimizer'] + del run_context_customized['net_outputs'] + del run_context_customized['loss_fn'] + net = WithLossCell(self.net, self.loss_fn) + net_cap = net + net_cap._cells = {'_backbone': self.net, + '_loss_fn': self.loss_fn} + net = TrainOneStep(net, self.optimizer) + net._cells = {'optimizer': self.optimizer, + 'network': net_cap, + 'backbone': self.net} + run_context_customized['train_network'] = net + train_callback.begin(RunContext(run_context_customized)) + train_callback.end(RunContext(run_context_customized)) + res = get_summary_lineage(SUMMARY_DIR) + assert res.get('hyper_parameters', {}).get('loss_function') \ + == 'SoftmaxCrossEntropyWithLogits' + assert res.get('algorithm', {}).get('network') == 'ResNet' + assert res.get('hyper_parameters', {}).get('optimizer') == 'Momentum' + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_raise_exception(self): + """Test exception when raise_exception is set True.""" + summary_record = SummaryRecord(SUMMARY_DIR_3) + full_file_name = summary_record.full_file_name + assert os.path.isfile(full_file_name) is True + assert os.path.isfile(full_file_name + "_lineage") is False + + train_callback = TrainLineage(summary_record, True) + eval_callback = EvalLineage(summary_record, False) + with self.assertRaises(LineageParamRunContextError): + train_callback.begin(self.run_context) + eval_callback.end(self.run_context) + file_num = os.listdir(SUMMARY_DIR_3) + assert len(file_num) == 1 + assert os.path.isfile(full_file_name + "_lineage") is False + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_raise_exception_init(self): + """Test exception when error happened during the initialization process.""" + if os.path.exists(SUMMARY_DIR_3): + shutil.rmtree(SUMMARY_DIR_3) + summary_record = SummaryRecord(SUMMARY_DIR_3) + train_callback = TrainLineage('fake_summary_record', False) + eval_callback = EvalLineage('fake_summary_record', False) + train_callback.begin(RunContext(self.run_context)) + eval_callback.end(RunContext(self.run_context)) + file_num = os.listdir(SUMMARY_DIR_3) + full_file_name = summary_record.full_file_name + assert len(file_num) == 1 + assert os.path.isfile(full_file_name + "_lineage") is False + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_raise_exception_create_file(self): + """Test exception when error happened after creating file.""" + if os.path.exists(SUMMARY_DIR_3): + shutil.rmtree(SUMMARY_DIR_3) + summary_record = SummaryRecord(SUMMARY_DIR_3) + eval_callback = EvalLineage(summary_record, False) + full_file_name = summary_record.full_file_name + "_lineage" + eval_run_context = self.run_context + eval_run_context['metrics'] = {'accuracy': 0.78} + eval_run_context['step_num'] = 32 + eval_run_context['valid_dataset'] = self.run_context['train_dataset'] + with open(full_file_name, 'ab'): + with mock.patch('builtins.open') as mock_handler: + mock_handler.return_value.__enter__.return_value.write.side_effect = IOError + eval_callback.end(RunContext(eval_run_context)) + assert os.path.isfile(full_file_name) is True + assert os.path.getsize(full_file_name) == 0 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_eval_run_context') + @mock.patch.object(AnalyzeObject, 'get_file_size', return_value=64) + def test_raise_exception_record_trainlineage(self, *args): + """Test exception when error happened after recording training infos.""" + if os.path.exists(SUMMARY_DIR_3): + shutil.rmtree(SUMMARY_DIR_3) + args[1].side_effect = MindInsightException(error=LineageErrors.PARAM_RUN_CONTEXT_ERROR, + message="RunContext error.") + summary_record = SummaryRecord(SUMMARY_DIR_3) + train_callback = TrainLineage(summary_record, True) + train_callback.begin(RunContext(self.run_context)) + full_file_name = summary_record.full_file_name + "_lineage" + file_size1 = os.path.getsize(full_file_name) + train_callback.end(RunContext(self.run_context)) + file_size2 = os.path.getsize(full_file_name) + assert file_size2 > file_size1 + eval_callback = EvalLineage(summary_record, False) + eval_callback.end(RunContext(self.run_context)) + file_size3 = os.path.getsize(full_file_name) + assert file_size3 == file_size2 + + @pytest.mark.level0 + @pytest.mark.platform_arm_ascend_training + @pytest.mark.platform_x86_gpu_training + @pytest.mark.platform_x86_ascend_training + @pytest.mark.platform_x86_cpu + @pytest.mark.env_single + def test_raise_exception_non_lineage_file(self): + """Test exception when lineage summary file cannot be found.""" + summary_dir = os.path.join(BASE_SUMMARY_DIR, 'run4') + if os.path.exists(summary_dir): + shutil.rmtree(summary_dir) + summary_record = SummaryRecord(summary_dir, file_suffix='_MS_lineage') + full_file_name = summary_record.full_file_name + assert full_file_name.endswith('_lineage') + assert os.path.isfile(full_file_name) + with self.assertRaisesRegex(LineageFileNotFoundError, 'no summary log file'): + get_summary_lineage(summary_dir) diff --git a/tests/st/func/lineagemgr/collection/model/train_one_step.py b/tests/st/func/lineagemgr/collection/model/train_one_step.py new file mode 100644 index 00000000..2595578f --- /dev/null +++ b/tests/st/func/lineagemgr/collection/model/train_one_step.py @@ -0,0 +1,28 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the customized network TrainOneStep.""" +import mindspore.nn as nn + + +class TrainOneStep(nn.Cell): + """Mock the customized network TrainOneStep.""" + def __init__(self, network, optimizer): + super(TrainOneStep, self).__init__(auto_prefix=False) + self.network = network + self.optimizer = optimizer + + def construct(self): + """Mock the construct method.""" + raise NotImplementedError diff --git a/tests/st/func/lineagemgr/conftest.py b/tests/st/func/lineagemgr/conftest.py new file mode 100644 index 00000000..04dc3d01 --- /dev/null +++ b/tests/st/func/lineagemgr/conftest.py @@ -0,0 +1,117 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The st config.""" + +import os +import shutil +import sys +import tempfile + +import pytest + +from .collection.model import mindspore + +sys.modules['mindspore'] = mindspore + +BASE_SUMMARY_DIR = tempfile.mkdtemp(prefix='test_lineage_summary_dir_base_') +SUMMARY_DIR = os.path.join(BASE_SUMMARY_DIR, 'run1') +SUMMARY_DIR_2 = os.path.join(BASE_SUMMARY_DIR, 'run2') +SUMMARY_DIR_3 = os.path.join(BASE_SUMMARY_DIR, 'except_run') + +COLLECTION_MODULE = 'TestModelLineage' +API_MODULE = 'TestModelApi' +DATASET_GRAPH = { + 'op_type': 'BatchDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'drop_remainder': True, + 'batch_size': 10, + 'children': [ + { + 'op_type': 'MapDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'input_columns': [ + 'label' + ], + 'output_columns': [ + None + ], + 'operations': [ + { + 'tensor_op_module': 'minddata.transforms.c_transforms', + 'tensor_op_name': 'OneHot', + 'num_classes': 10 + } + ], + 'children': [ + { + 'op_type': 'MnistDataset', + 'shard_id': None, + 'num_shards': None, + 'op_module': 'minddata.dataengine.datasets', + 'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData', + 'num_parallel_workers': None, + 'shuffle': None, + 'num_samples': 100, + 'sampler': { + 'sampler_module': 'minddata.dataengine.samplers', + 'sampler_name': 'RandomSampler', + 'replacement': True, + 'num_samples': 100 + }, + 'children': [] + } + ] + } + ] +} + +def get_module_name(nodeid): + """Get the module name from nodeid.""" + _, module_name, _ = nodeid.split("::") + return module_name + + +def pytest_collection_modifyitems(items): + """Modify the execution order.""" + split_items = { + COLLECTION_MODULE: [], + API_MODULE: [] + } + for item in items: + module_name = get_module_name(item.nodeid) + module_item = split_items.get(module_name) + if module_item is not None: + module_item.append(item) + ordered_items = split_items.get(COLLECTION_MODULE) + ordered_items.extend(split_items.get(API_MODULE)) + items[:] = ordered_items + + +@pytest.fixture(scope="session") +def create_summary_dir(): + """Create summary directory.""" + try: + if os.path.exists(BASE_SUMMARY_DIR): + shutil.rmtree(BASE_SUMMARY_DIR) + permissions = os.R_OK | os.W_OK | os.X_OK + mode = permissions << 6 + if not os.path.exists(BASE_SUMMARY_DIR): + os.mkdir(BASE_SUMMARY_DIR, mode=mode) + yield + finally: + if os.path.exists(BASE_SUMMARY_DIR): + shutil.rmtree(BASE_SUMMARY_DIR) diff --git a/tests/st/runtest.sh b/tests/st/runtest.sh new file mode 100644 index 00000000..91d774c2 --- /dev/null +++ b/tests/st/runtest.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +shopt -s nullglob + +SCRIPT_BASEDIR=$( + cd "$(dirname "$0")" || exit + pwd +) +PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") +CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh" +CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/" +ST_PATH="${PROJECT_DIR}/tests/st" +IS_BUILD_CRC="" + +PYTEST_MARK="" + +show_usage() { + echo "Run test cases for st." + echo "" + echo "usage: runtest.sh [-h] [-m ]" + echo "" + echo "options:" + echo " -h show usage info" + echo " -m mark of pytest." + +} + +check_opts() { + while getopts ":hm:" opt; do + case $opt in + h) + show_usage + exit 0 + ;; + m) + PYTEST_MARK="$OPTARG" + ;; + \?) + show_usage + exit 1 + ;; + esac + done +} + +build_crc32() { + echo "Start to check crc32." + if [ -d "$CRC32_OUTPUT_DIR" ]; then + cd "$CRC32_OUTPUT_DIR" + result=$(find . -maxdepth 1 -name "crc32*.so") + if [ -z "$result" ]; then + echo "Start to build crc32." + IS_BUILD_CRC="true" + bash "$CRC32_SCRIPT_PATH" + fi + fi + +} + +clean_crc32() { + echo "Start to clean crc32." + if [ -n "$IS_BUILD_CRC" ]; then + rm "$CRC32_OUTPUT_DIR"/crc32*.so -f + fi +} + +before_run_test() { + echo "Before run tests." + export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH + build_crc32 +} + +after_run_test() { + echo "After run tests." + clean_crc32 + + echo "End to run tests." +} + +run_test() { + echo "Start to run test." + cd "$PROJECT_DIR" + + for dir in "$ST_PATH"/*; do + if [ ! -d "$dir" ] || [ "$dir" = "$ST_PATH/__pycache__" ]; then + continue + fi + + for sub_dir in "$dir"/*; do + if [ ! -d "$sub_dir" ] || [ "$sub_dir" = "$dir/__pycache__" ]; then + continue + fi + echo "Run test for path: $sub_dir" + pytest "$sub_dir" --disable-pytest-warnings -m "$PYTEST_MARK" + done + done + + echo "Test all use cases success." +} + +check_opts "$@" +before_run_test +run_test +after_run_test diff --git a/tests/ut/__init__.py b/tests/ut/__init__.py new file mode 100644 index 00000000..4db1e497 --- /dev/null +++ b/tests/ut/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Import the mocked mindspore.""" +import sys +from .lineagemgr.collection.model import mindspore + +sys.modules['mindspore'] = mindspore diff --git a/tests/ut/backend/__init__.py b/tests/ut/backend/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/backend/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/backend/datavisual/__init__.py b/tests/ut/backend/datavisual/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/backend/datavisual/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/backend/datavisual/conftest.py b/tests/ut/backend/datavisual/conftest.py new file mode 100644 index 00000000..431dc03e --- /dev/null +++ b/tests/ut/backend/datavisual/conftest.py @@ -0,0 +1,58 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Description: This file is used for some common util. +""" +from unittest.mock import Mock + +import pytest +from flask import Response + +from mindinsight.backend import datavisual +from mindinsight.datavisual import utils + + +@pytest.fixture +def client(): + """This fixture is flask client.""" + mock_data_manager = Mock() + mock_data_manager.start_load_data = Mock() + datavisual.DATA_MANAGER = mock_data_manager + + packages = ["mindinsight.backend.raw_dataset", + "mindinsight.backend.train_dataset", + "mindinsight.backend.data_visual"] + + mock_obj = Mock(return_value=packages) + utils.find_app_package = mock_obj + + from mindinsight.backend.application import APP + APP.response_class = Response + app_client = APP.test_client() + + yield app_client + + +TRAIN_ROUTES = dict( + train_jobs='/v1/mindinsight/datavisual/train-jobs', + single_job='/v1/mindinsight/datavisual/single-job', + plugins='/v1/mindinsight/datavisual/plugins', + graph_nodes='/v1/mindinsight/datavisual/graphs/nodes', + graph_nodes_names='/v1/mindinsight/datavisual/graphs/nodes/names', + graph_single_node='/v1/mindinsight/datavisual/graphs/single-node', + image_metadata='/v1/mindinsight/datavisual/image/metadata', + image_single_image='/v1/mindinsight/datavisual/image/single-image', + scalar_metadata='/v1/mindinsight/datavisual/scalar/metadata' +) diff --git a/tests/ut/backend/datavisual/test_task_manager_api.py b/tests/ut/backend/datavisual/test_task_manager_api.py new file mode 100644 index 00000000..8beaff41 --- /dev/null +++ b/tests/ut/backend/datavisual/test_task_manager_api.py @@ -0,0 +1,187 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test train task restful api. +Usage: + pytest tests/ut/datavisual +""" +from unittest.mock import patch + +import pytest +from tests.ut.backend.datavisual.conftest import TRAIN_ROUTES +from tests.ut.datavisual.utils.log_generators.images_log_generator import ImagesLogGenerator +from tests.ut.datavisual.utils.log_generators.scalars_log_generator import ScalarsLogGenerator +from tests.ut.datavisual.utils.utils import get_url + +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.processors.train_task_manager import TrainTaskManager + + +class TestTrainTask: + """Test train task api.""" + + _scalar_log_generator = ScalarsLogGenerator() + _image_log_generator = ImagesLogGenerator() + + @pytest.mark.parametrize( + "plugin_name", + ['no_plugin_name', 'not_exist_plugin_name']) + def test_query_single_train_task_with_plugin_name_not_exist(self, client, plugin_name): + """ + Parsing unavailable plugin name to single train task. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/single-job'). + request params: plugin_name. + + Expect: + response status code: 400. + response json: { + 'error_code': '50540002', + 'error_message': "Invalid parameter value. 'plugin_name' only can + be one of ['graph', 'image', 'scalar']" + } + """ + plugin_name_list = PluginNameEnum.list_members() + + params = dict(plugin_name=plugin_name, train_id="not_exist") + url = get_url(TRAIN_ROUTES['single_job'], params) + + response = client.get(url) + + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. 'plugin_name' " \ + "only can be one of {}".format(plugin_name_list) + + @patch.object(TrainTaskManager, 'get_single_train_task') + def test_query_single_train_task_with_plugin_name_exists(self, mock_train_task_manager, client): + """ + Parsing unavailable plugin name to get single train task. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/single-job'). + request params: plugin_name. + + Expect: + response status code: 200. + response json: plugin_name and train_id. + """ + + def get_single_train_task(plugin_name, train_id): + return f'{plugin_name}{train_id}' + + mock_train_task_manager.side_effect = get_single_train_task + + plugin_name = PluginNameEnum.IMAGE.value + train_id = "test_id" + + params = dict(plugin_name=plugin_name, train_id=train_id) + url = get_url(TRAIN_ROUTES['single_job'], params) + + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == f'{plugin_name}{train_id}' + + def test_query_single_train_task_with_param_miss(self, client): + """ + Parsing unavailable plugin name to get single train task. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/single-job'). + request params: plugin_name. + + Expect: + response status code: 400. + response json: { + 'error_code': '50540003', + 'error_message': "Param missing. 'plugin_name' is required." + } + """ + params = dict() + url = get_url(TRAIN_ROUTES['single_job'], params) + + response = client.get(url) + + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'plugin_name' is required." + + def test_query_plugins_with_manual_update_wrong(self, client): + """ + Parsing unavailable manual_update to get plugins. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/plugins'). + request params: manual_update. + + Expect: + response status code: 400. + response json: { + 'error_code': '50540002', + 'error_message': "Invalid parameter value. The value of manual_update + must be 'false' or 'true'." + } + """ + params = dict(train_id="test_id", manual_update="hhh") + url = get_url(TRAIN_ROUTES['plugins'], params) + + response = client.get(url) + + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540002' + assert response['error_msg'] == "Invalid parameter value. The value of manual_update " \ + "must be 'false' or 'true'." + + @patch.object(TrainTaskManager, 'get_plugins') + def test_query_plugins_success(self, mock_train_task_manager, client): + """ + Parsing unavailable run plugin to get train jobs. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/train-plugins'). + request params: plugin_name. + request params: manual_update. + + Expect: + response status code: 200. + response json: plugin_name. + """ + + def get_plugins(train_id, manual_update): + update_str = "false" + if manual_update: + update_str = "true" + return f'{train_id}{update_str}' + + train_id = "test_id" + manual_update = "true" + mock_train_task_manager.side_effect = get_plugins + + params = dict(train_id=train_id, manual_update=manual_update) + url = get_url(TRAIN_ROUTES['plugins'], params) + + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == f'{train_id}{manual_update}' diff --git a/tests/ut/backend/datavisual/test_train_visual_api.py b/tests/ut/backend/datavisual/test_train_visual_api.py new file mode 100644 index 00000000..1ef85910 --- /dev/null +++ b/tests/ut/backend/datavisual/test_train_visual_api.py @@ -0,0 +1,444 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test train visual api. +Usage: + pytest tests/ut/datavisual +""" +from unittest.mock import Mock, patch + +import pytest +from tests.ut.backend.datavisual.conftest import TRAIN_ROUTES +from tests.ut.datavisual.utils.utils import get_url + +from mindinsight.datavisual.data_transform.graph import NodeTypeEnum +from mindinsight.datavisual.processors.graph_processor import GraphProcessor +from mindinsight.datavisual.processors.images_processor import ImageProcessor +from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor + + +class TestTrainVisual: + """Test Train Visual APIs.""" + + def test_image_metadata_with_params_miss(self, client): + """Test missing params when getting image metadata.""" + + params = dict() + url = get_url(TRAIN_ROUTES['image_metadata'], params) + + response = client.get(url) + + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'train_id' is required." + + train_id = "aa" + + params = dict(train_id=train_id) + url = get_url(TRAIN_ROUTES['image_metadata'], params) + + response = client.get(url) + + assert response.status_code == 400 + + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'tag' is required." + + @patch.object(ImageProcessor, 'get_metadata_list') + def test_image_metadata_with_params_success(self, mock_processor, client): + """ + Parsing available params to get image metadata. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/image/metadata'). + request params: train_id. + request params: tag. + + Expect: + response status code: 400. + response json: { + 'error_code': '50545006', + 'error_message': "Invalid parameter value. 'plugin_name' only can + be one of ['graph', 'image', 'scalar']" + }. + """ + mock_get_metadata_list = Mock() + mock_get_metadata_list.return_value = { + "metadatas": [{ + "height": 224, + "step": 1, + "wall_time": 1572058058.1175, + "width": 448 + }] + } + params = dict(train_id='123', tag='123') + url = get_url(TRAIN_ROUTES['image_metadata'], params) + + mock_processor.side_effect = mock_get_metadata_list + response = client.get(url) + + assert response.status_code == 200 + response = response.get_json() + expected_response = { + "metadatas": [{ + "height": 224, + "step": 1, + "wall_time": 1572058058.1175, + "width": 448 + }] + } + assert expected_response == response + + def test_single_image_with_params_miss(self, client): + """ + Test missing params when getting single image. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/image/single-image'). + request params: train_id. + request params: tag. + request params: step. + + Expect: + response status code: 400. + response json: { + 'error_code': '50540003', + 'error_message': "Param missing. 'train_id' is required." + }. + """ + params = dict() + url = get_url(TRAIN_ROUTES['image_single_image'], params) + response = client.get(url) + assert response.status_code == 400 + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'train_id' is required." + + params = dict(train_id='123') + url = get_url(TRAIN_ROUTES['image_single_image'], params) + response = client.get(url) + assert response.status_code == 400 + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'tag' is required." + + params = dict(train_id='123', tag='456') + url = get_url(TRAIN_ROUTES['image_single_image'], params) + response = client.get(url) + assert response.status_code == 400 + response = response.get_json() + assert response['error_code'] == '50540003' + assert response['error_msg'] == "Param missing. 'step' is required." + + def test_single_image_with_step_not_int(self, client): + """ + Test the `tag` param is not int when getting single image. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/image/single-image'). + request params: train_id. + request params: tag. + request params: step. + + Expect: + response status code: 400. + response json: { + 'error_code': '50540001', + 'error_message': "Invalid parameter type. 'step' expect Integer type." + }. + """ + params = dict(train_id='123', tag='456', step='abc') + url = get_url(TRAIN_ROUTES['image_single_image'], params) + response = client.get(url) + assert response.status_code == 400 + response = response.get_json() + assert response['error_code'] == '50540001' + assert response['error_msg'] == "Invalid parameter type. 'step' expect Integer type." + + @patch.object(ImageProcessor, 'get_single_image') + def test_single_image_with_params_success(self, mock_processor, client): + """Test getting single image with params successfully.""" + mock_get_single_image = Mock(return_value=b'123') + params = dict(train_id='123', tag='123', step=1) + url = get_url(TRAIN_ROUTES['image_single_image'], params) + + mock_processor.side_effect = mock_get_single_image + response = client.get(url) + + assert response.status_code == 200 + assert response.data == b'123' + + def test_scalar_metadata_with_params_is_none(self, client): + """Parsing unavailable params to get scalar metadata.""" + params = dict() + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'train_id' is required." + + train_id = "aa" + params = dict(train_id=train_id) + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + response = client.get(url) + results = response.get_json() + + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'tag' is required." + + @patch.object(ScalarsProcessor, 'get_metadata_list') + def test_scalar_metadata_success(self, mock_scalar_processor, client): + """Parsing available params to get scalar metadata.""" + get_metadata_list = Mock(return_value={'metadatas': [{'value': 1}]}) + mock_scalar_processor.side_effect = get_metadata_list + + test_train_id = "aa" + test_tag = "bb" + params = dict(train_id=test_train_id, tag=test_tag) + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == dict(metadatas=[dict(value=1)]) + + def test_graph_nodes_with_train_id_is_none(self, client): + """Test getting graph nodes when train_id is none.""" + params = dict() + url = get_url(TRAIN_ROUTES['graph_nodes'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'train_id' is required." + + @patch.object(GraphProcessor, '__init__') + def test_graph_nodes_with_type_is_invalid(self, mock_graph_processor, client): + """Test getting graph nodes with invalid type.""" + mock_init = Mock(return_value=None) + mock_graph_processor.side_effect = mock_init + + node_type = "invalid_node_type" + params = dict(train_id='aaa', type=node_type) + url = get_url(TRAIN_ROUTES['graph_nodes'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540002' + assert results['error_msg'] == "Invalid parameter value. The node type " \ + "is not support, only either %s or %s." \ + "" % (NodeTypeEnum.NAME_SCOPE.value, + NodeTypeEnum.POLYMERIC_SCOPE.value) + + @patch.object(GraphProcessor, '__init__') + @patch.object(GraphProcessor, 'get_nodes') + def test_graph_nodes_success(self, mock_graph_processor, mock_graph_processor_1, client): + """Test getting graph nodes successfully.""" + def mock_get_nodes(name, node_type): + return dict(name=name, node_type=node_type) + mock_graph_processor.side_effect = mock_get_nodes + + mock_init = Mock(return_value=None) + mock_graph_processor_1.side_effect = mock_init + + test_train_id = 'aaa' + test_node_name = 'bbb' + test_node_type = NodeTypeEnum.NAME_SCOPE.value + params = dict(train_id=test_train_id, name=test_node_name, type=test_node_type) + url = get_url(TRAIN_ROUTES['graph_nodes'], params) + + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == dict(name=test_node_name, node_type=test_node_type) + + def test_graph_node_names_with_train_id_is_none(self, client): + """Test getting graph node names with train id is none.""" + params = dict() + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'train_id' is required." + + @patch.object(GraphProcessor, '__init__') + def test_graph_node_names_with_param_type(self, mock_graph_processor, client): + """Test getting graph node names with param type.""" + mock_init = Mock(return_value=None) + mock_graph_processor.side_effect = mock_init + + test_train_id = 'aaa' + params = dict(train_id=test_train_id, offset='a', limit='b') + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540001' + assert results['error_msg'] == "Invalid parameter type. 'offset' expect Integer type." + + params = dict(train_id=test_train_id, offset=1, limit='c') + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540001' + assert results['error_msg'] == "Invalid parameter type. 'limit' expect Integer type." + + @patch.object(GraphProcessor, '__init__') + def test_graph_node_names_with_invalid_offset(self, mock_graph_processor, client): + """Test getting graph node names with invalid offset.""" + mock_init = Mock(return_value=None) + mock_graph_processor.side_effect = mock_init + + test_train_id = 'aaa' + test_offset = -1 + test_limit = 100 + params = dict(train_id=test_train_id, offset=test_offset, limit=test_limit) + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540002' + assert results['error_msg'] == "Invalid parameter value. 'offset' should " \ + "be greater than or equal to 0." + + @pytest.mark.parametrize( + "limit", + [-1, 0, 1001] + ) + @patch.object(GraphProcessor, '__init__') + def test_graph_node_names_with_invalid_limit(self, mock_graph_processor, client, limit): + """Test getting graph node names with invalid limit.""" + mock_init = Mock(return_value=None) + mock_graph_processor.side_effect = mock_init + + test_train_id = 'aaa' + params = dict(train_id=test_train_id, limit=limit) + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540002' + assert results['error_msg'] == "Invalid parameter value. " \ + "'limit' should in [1, 1000]." + + @pytest.mark.parametrize( + " offset, limit", + [(0, 100), (1, 1), (0, 1000)] + ) + @patch.object(GraphProcessor, '__init__') + @patch.object(GraphProcessor, 'search_node_names') + def test_graph_node_names_success(self, mock_graph_processor, mock_graph_processor_1, client, + offset, limit): + """ + Parsing unavailable params to get image metadata. + + Test Params: + request route: GET('/v1/mindinsight/datavisual/graph/nodes/names'). + request params: plugin_name. + + Expect: + response status code: 200. + response json: dict, contains search_content, offset, and limit. + """ + def mock_search_node_names(search_content, offset, limit): + return dict(search_content=search_content, offset=int(offset), limit=int(limit)) + mock_graph_processor.side_effect = mock_search_node_names + + mock_init = Mock(return_value=None) + mock_graph_processor_1.side_effect = mock_init + + test_train_id = "aaa" + test_search_content = "bbb" + params = dict(train_id=test_train_id, search=test_search_content, + offset=offset, limit=limit) + url = get_url(TRAIN_ROUTES['graph_nodes_names'], params) + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == dict(search_content=test_search_content, + offset=int(offset), + limit=int(limit)) + + def test_graph_search_single_node_with_params_is_wrong(self, client): + """Test searching graph single node with params is wrong.""" + test_name = 'aaa' + params = dict(name=test_name) + url = get_url(TRAIN_ROUTES['graph_single_node'], params) + + response = client.get(url) + results = response.get_json() + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'train_id' is required." + + @patch.object(GraphProcessor, '__init__') + def test_graph_search_single_node_with_params_is_none(self, mock_graph_processor, client): + """Test searching graph single node with node_name is none.""" + mock_init = Mock(return_value=None) + + mock_graph_processor.side_effect = mock_init + params = dict() + url = get_url(TRAIN_ROUTES['graph_single_node'], params) + response = client.get(url) + results = response.get_json() + + assert response.status_code == 400 + assert results['error_code'] == '50540003' + assert results['error_msg'] == "Param missing. 'name' is required." + + @patch.object(GraphProcessor, '__init__') + @patch.object(GraphProcessor, 'search_single_node') + def test_graph_search_single_node_success(self, mock_graph_processor, mock_graph_processor_1, client): + """ + Test searching graph single node successfully. + + Test Params: + request route: GET('/v1/mindinsight/datavisual//'). + request params: plugin_name. + + Expect: + response status code: 200. + response json: name. + """ + def mock_search_single_node(name): + return name + mock_graph_processor.side_effect = mock_search_single_node + + mock_init = Mock(return_value=None) + mock_graph_processor_1.side_effect = mock_init + + test_train_id = "aaa" + test_name = 'bbb' + params = dict(train_id=test_train_id, name=test_name) + url = get_url(TRAIN_ROUTES['graph_single_node'], params) + response = client.get(url) + assert response.status_code == 200 + results = response.get_json() + assert results == test_name diff --git a/tests/ut/backend/lineagemgr/__init__.py b/tests/ut/backend/lineagemgr/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/backend/lineagemgr/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/backend/lineagemgr/test_lineage_api.py b/tests/ut/backend/lineagemgr/test_lineage_api.py new file mode 100644 index 00000000..8c3dfef3 --- /dev/null +++ b/tests/ut/backend/lineagemgr/test_lineage_api.py @@ -0,0 +1,161 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the module of lineage_api.""" +import json +import os +from unittest import TestCase, mock + +from flask import Response + +from mindinsight.backend.application import APP +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageQuerySummaryDataError + + +class TestSearchModel(TestCase): + """Test the restful api of search_model.""" + + def setUp(self): + """Test init.""" + APP.response_class = Response + self.app_client = APP.test_client() + self.url = '/v1/mindinsight/models/model_lineage' + + @mock.patch('mindinsight.backend.lineagemgr.lineage_api.settings') + @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage') + def test_search_model_success(self, *args): + """Test the success of model_success.""" + base_dir = '/path/to/test_lineage_summary_dir_base' + args[0].return_value = { + 'object': [ + { + 'summary_dir': base_dir, + 'accuracy': None, + 'mae': None, + 'mse': None, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 64, + 'test_dataset_path': None, + 'test_dataset_count': None, + 'network': 'str', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 12, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 128 + }, + { + 'summary_dir': os.path.join(base_dir, 'run1'), + 'accuracy': 0.78, + 'mae': None, + 'mse': None, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 64, + 'test_dataset_path': None, + 'test_dataset_count': 64, + 'network': 'str', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 128 + } + ], + 'count': 2 + } + args[1].SUMMARY_BASE_DIR = base_dir + + body_data = { + 'limit': 10, + 'offset': 0, + 'sorted_name': 'summary_dir', + 'sorted_type': None + } + response = self.app_client.post(self.url, data=json.dumps(body_data)) + self.assertEqual(200, response.status_code) + expect_result = { + 'object': [ + { + 'summary_dir': './', + 'accuracy': None, + 'mae': None, + 'mse': None, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 64, + 'test_dataset_path': None, + 'test_dataset_count': None, + 'network': 'str', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 12, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 128 + }, + { + 'summary_dir': './run1', + 'accuracy': 0.78, + 'mae': None, + 'mse': None, + 'loss_function': 'SoftmaxCrossEntropyWithLogits', + 'train_dataset_path': None, + 'train_dataset_count': 64, + 'test_dataset_path': None, + 'test_dataset_count': 64, + 'network': 'str', + 'optimizer': 'Momentum', + 'learning_rate': 0.11999999731779099, + 'epoch': 14, + 'batch_size': 32, + 'loss': 0.029999999329447746, + 'model_size': 128 + } + ], + 'count': 2 + } + self.assertDictEqual(expect_result, response.get_json()) + + @mock.patch('mindinsight.backend.lineagemgr.lineage_api.settings') + @mock.patch('mindinsight.backend.lineagemgr.lineage_api.filter_summary_lineage') + def test_search_model_fail(self, *args): + """Test the function of model_lineage with exception.""" + response = self.app_client.post(self.url, data='xxx') + self.assertEqual(400, response.status_code) + expect_result = { + 'error_code': '50540002', + 'error_msg': 'Invalid parameter value. Json data parse failed.' + } + self.assertDictEqual(expect_result, response.get_json()) + + args[0].side_effect = LineageQuerySummaryDataError('xxx') + args[1].SUMMARY_BASE_DIR = '/path/to/test_lineage_summary_dir_base' + body_data = { + 'limit': 10, + 'offset': 0, + 'sorted_name': 'summary_dir', + 'sorted_type': None + } + response = self.app_client.post(self.url, data=json.dumps(body_data)) + self.assertEqual(400, response.status_code) + expect_result = { + 'error_code': '50542215', + 'error_msg': 'Query summary data error: xxx' + } + self.assertDictEqual(expect_result, response.get_json()) diff --git a/tests/ut/datavisual/__init__.py b/tests/ut/datavisual/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/common/__init__.py b/tests/ut/datavisual/common/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/common/test_error_handler.py b/tests/ut/datavisual/common/test_error_handler.py new file mode 100644 index 00000000..58c13683 --- /dev/null +++ b/tests/ut/datavisual/common/test_error_handler.py @@ -0,0 +1,102 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test error handler. +Usage: + pytest tests/ut/datavisual +""" +from unittest.mock import patch +from werkzeug.exceptions import MethodNotAllowed, NotFound + +from tests.ut.backend.datavisual.conftest import TRAIN_ROUTES +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.utils import get_url + +from mindinsight.datavisual.processors import scalars_processor +from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor + + +class TestErrorHandler: + """Test train visual api.""" + + @patch.object(ScalarsProcessor, 'get_metadata_list') + def test_handle_http_exception_error_not_found(self, mock_scalar_processor, client): + """Test handle http exception error not found.""" + scalars_processor.logger = MockLogger + text = 'Test Message' + + # NotFound + def get_metadata_list(train_ids, tag): + raise NotFound("%s" % text) + + mock_scalar_processor.side_effect = get_metadata_list + + test_train_ids = "aa" + test_tag = "bb" + params = dict(train_ids=test_train_ids, tag=test_tag) + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + response = client.get(url) + + assert response.status_code == 404 + response = response.get_json() + assert response['error_code'] == '50545001' + assert response['error_msg'] == '404 Not Found.' + + @patch.object(ScalarsProcessor, 'get_metadata_list') + def test_handle_http_exception_error_method_not_allowed(self, mock_scalar_processor, client): + """Test handling http exception error method not allowed.""" + scalars_processor.logger = MockLogger + text = 'Test Message' + + # MethodNotAllowed + def get_metadata_list(train_ids, tag): + raise MethodNotAllowed("%s" % text) + + mock_scalar_processor.side_effect = get_metadata_list + + test_train_ids = "aa" + test_tag = "bb" + params = dict(train_ids=test_train_ids, tag=test_tag) + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + response = client.get(url) + + assert response.status_code == 405 + response = response.get_json() + assert response['error_code'] == '50545002' + assert response['error_msg'] == '405 Method Not Allowed.' + + @patch.object(ScalarsProcessor, 'get_metadata_list') + def test_handle_http_exception_error_method_other_errors(self, mock_scalar_processor, client): + """Test handling http exception error method other errors.""" + scalars_processor.logger = MockLogger + text = 'Test Message' + + # Other errors + def get_metadata_list(train_ids, tag): + raise KeyError("%s" % text) + + mock_scalar_processor.side_effect = get_metadata_list + + test_train_ids = "aa" + test_tag = "bb" + params = dict(train_ids=test_train_ids, tag=test_tag) + url = get_url(TRAIN_ROUTES['scalar_metadata'], params) + response = client.get(url) + + assert response.status_code == 500 + response = response.get_json() + assert response['error_code'] == '50540000' + assert response['error_msg'] == 'System error.' diff --git a/tests/ut/datavisual/conftest.py b/tests/ut/datavisual/conftest.py new file mode 100644 index 00000000..fd4120e6 --- /dev/null +++ b/tests/ut/datavisual/conftest.py @@ -0,0 +1,45 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Description: This file is used for some common util. +""" +from unittest.mock import Mock + +import pytest +from flask import Response + +from mindinsight.backend import datavisual +from mindinsight.datavisual import utils + + +@pytest.fixture +def client(): + """This fixture is flask client.""" + mock_data_manager = Mock() + mock_data_manager.start_load_data = Mock() + datavisual.DATA_MANAGER = mock_data_manager + + packages = ["mindinsight.backend.raw_dataset", + "mindinsight.backend.train_dataset", + "mindinsight.backend.data_visual"] + + mock_obj = Mock(return_value=packages) + utils.find_app_package = mock_obj + + from mindinsight.backend.application import APP + APP.response_class = Response + app_client = APP.test_client() + + yield app_client diff --git a/tests/ut/datavisual/data_transform/__init__.py b/tests/ut/datavisual/data_transform/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/data_transform/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/data_transform/loader_generators/__init__.py b/tests/ut/datavisual/data_transform/loader_generators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ut/datavisual/data_transform/loader_generators/test_data_loader_generator.py b/tests/ut/datavisual/data_transform/loader_generators/test_data_loader_generator.py new file mode 100644 index 00000000..eab728de --- /dev/null +++ b/tests/ut/datavisual/data_transform/loader_generators/test_data_loader_generator.py @@ -0,0 +1,124 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.log_generators.data_loader_generator +Usage: + pytest tests/ut/datavisual +""" +import datetime +import os +import shutil +import tempfile + +from unittest.mock import patch +import pytest + +from tests.ut.datavisual.mock import MockLogger + +from mindinsight.datavisual.data_transform.loader_generators import data_loader_generator +from mindinsight.utils.exceptions import ParamValueError + + +class TestDataLoaderGenerator: + """Test data_loader_generator.""" + @classmethod + def setup_class(cls): + data_loader_generator.logger = MockLogger + + def _generate_summaries(self, summary_base_dir, dir_num=1): + """Utils function for tests.""" + summaries = list() + + if os.path.exists(summary_base_dir): + shutil.rmtree(summary_base_dir) + os.mkdir(summary_base_dir) + for i in range(dir_num): + log_dir = os.path.join(summary_base_dir, f'job{i}') + os.mkdir(log_dir) + summary_info = dict(relative_path=log_dir.replace(summary_base_dir, "."), + update_time=datetime.datetime.now().replace(minute=i).astimezone()) + summaries.append(summary_info) + return summaries + + def test_invalid_summary_path(self): + """Test invalid summary path.""" + path = None + with pytest.raises(ParamValueError) as exc_info: + data_loader_generator.DataLoaderGenerator(path) + + assert "Summary path is None." in exc_info.value.message + assert MockLogger.log_msg['warning'] == "Summary path is None. It will not init data loader generator." + + def test_generate_loaders_with_not_exist_path(self): + """Test generating loaders with not exist path.""" + path = tempfile.NamedTemporaryFile().name + os.mkdir(path) + generator = data_loader_generator.DataLoaderGenerator(path) + os.removedirs(path) + loader_dict = generator.generate_loaders(loader_pool=dict()) + assert MockLogger.log_msg['warning'] == "Summary path does not exist. It will not start " \ + "loading events data. Current path is %r." % path + assert loader_dict == {} + + @patch.object(data_loader_generator.DataLoader, "has_valid_files") + @patch.object(data_loader_generator.SummaryWatcher, "list_summary_directories") + def test_generate_loaders(self, mock_summary_watcher, mock_data_loader): + """Test generating loaders.""" + summary_base_dir = tempfile.NamedTemporaryFile().name + summaries = self._generate_summaries(summary_base_dir, 20) + + # mock summary_watcher. + generator = data_loader_generator.DataLoaderGenerator(summary_base_dir) + mock_summary_watcher.return_value = summaries + + # mock DataLoader + mock_data_loader.return_value = True + + loader_dict = generator.generate_loaders(loader_pool=dict()) + expected_ids = [summary.get('relative_path') + for summary in summaries[-data_loader_generator.MAX_DATA_LOADER_SIZE:]] + assert sorted(loader_dict.keys()) == sorted(expected_ids) + + shutil.rmtree(summary_base_dir) + + def test_check_job_exist(self): + """Test checking if job exists.""" + summary_base_dir = tempfile.NamedTemporaryFile().name + self._generate_summaries(summary_base_dir) + with open(os.path.join(summary_base_dir, 'job0', "summary.1"), 'w'): + pass + + generator = data_loader_generator.DataLoaderGenerator(summary_base_dir) + + assert generator.check_train_job_exist(train_id='./job0') + assert not generator.check_train_job_exist(train_id='job0') + assert not generator.check_train_job_exist(train_id='././../job0') + + shutil.rmtree(summary_base_dir) + + def test_generate_loader_by_train_id(self): + """Test generating loader by train id.""" + summary_base_dir = tempfile.NamedTemporaryFile().name + self._generate_summaries(summary_base_dir) + with open(os.path.join(summary_base_dir, 'job0', "summary.1"), 'w'): + pass + generator = data_loader_generator.DataLoaderGenerator(summary_base_dir) + + train_id = "./job0" + loader = generator.generate_loader_by_train_id(train_id) + assert loader.loader_id == train_id + + shutil.rmtree(summary_base_dir) diff --git a/tests/ut/datavisual/data_transform/test_data_loader.py b/tests/ut/datavisual/data_transform/test_data_loader.py new file mode 100644 index 00000000..13624fa4 --- /dev/null +++ b/tests/ut/datavisual/data_transform/test_data_loader.py @@ -0,0 +1,81 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.data_loader. +Usage: + pytest tests/ut/datavisual +""" +import os +import shutil +import tempfile + +import pytest +from tests.ut.datavisual.mock import MockLogger + +from mindinsight.datavisual.common.exceptions import SummaryLogPathInvalid +from mindinsight.datavisual.data_transform import data_loader +from mindinsight.datavisual.data_transform.data_loader import DataLoader + + +class TestDataLoader: + """Test data_loader.""" + + @classmethod + def setup_class(cls): + data_loader.logger = MockLogger + + def setup_method(self, method): + self._summary_dir = tempfile.mkdtemp() + if os.path.exists(self._summary_dir): + shutil.rmtree(self._summary_dir) + os.mkdir(self._summary_dir) + + def teardown_method(self, method): + if os.path.exists(self._summary_dir): + shutil.rmtree(self._summary_dir) + + def _generate_files(self, dir_path, file_list): + for file_name in file_list: + with open(os.path.join(dir_path, file_name), 'w'): + pass + + def test_load_with_not_file_list(self): + """Test loading method with empty file list.""" + loader = DataLoader(self._summary_dir) + with pytest.raises(SummaryLogPathInvalid): + loader.load() + assert 'No valid files can be loaded' in str(MockLogger.log_msg['warning']) + + def test_load_with_invalid_file_list(self): + """Test loading method with valid path and invalid file_list.""" + file_list = ['summary.abc01', 'summary.abc02'] + self._generate_files(self._summary_dir, file_list) + loader = DataLoader(self._summary_dir) + with pytest.raises(SummaryLogPathInvalid): + loader.load() + assert 'No valid files can be loaded' in str(MockLogger.log_msg['warning']) + + def test_load_success(self): + """Test loading method with valid path and file_list.""" + dir_path = tempfile.NamedTemporaryFile().name + if not os.path.exists(dir_path): + os.mkdir(dir_path) + file_list = ['summary.001', 'summary.002'] + self._generate_files(dir_path, file_list) + dataloader = DataLoader(dir_path) + dataloader.load() + assert dataloader._loader is not None + shutil.rmtree(dir_path) diff --git a/tests/ut/datavisual/data_transform/test_data_manager.py b/tests/ut/datavisual/data_transform/test_data_manager.py new file mode 100644 index 00000000..19a1fc64 --- /dev/null +++ b/tests/ut/datavisual/data_transform/test_data_manager.py @@ -0,0 +1,224 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.data_manager. +Usage: + pytest tests/ut/datavisual +""" +import time +import os +import shutil +import tempfile +from unittest import mock +from unittest.mock import Mock +from unittest.mock import patch + +import pytest +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.utils import check_loading_done + +from mindinsight.datavisual.common.enums import DataManagerStatus, PluginNameEnum +from mindinsight.datavisual.data_transform import data_manager, ms_data_loader +from mindinsight.datavisual.data_transform.data_loader import DataLoader +from mindinsight.datavisual.data_transform.data_manager import DataManager +from mindinsight.datavisual.data_transform.events_data import EventsData +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import \ + DataLoaderGenerator +from mindinsight.datavisual.data_transform.loader_generators.loader_generator import \ + MAX_DATA_LOADER_SIZE +from mindinsight.datavisual.data_transform.loader_generators.loader_struct import \ + LoaderStruct +from mindinsight.datavisual.data_transform.ms_data_loader import MSDataLoader +from mindinsight.utils.exceptions import ParamValueError + + +class TestDataManager: + """Test data_manager.""" + _plugin_name = PluginNameEnum.IMAGE.value + _train_id_suffix = "_test_data_manager" + + @pytest.fixture(scope="function") + def crc_pass(self): + """Mock the crc to pass the check.""" + ms_data_loader.crc32.GetValueFromStr = Mock(return_value=0) + ms_data_loader.crc32.GetMaskCrc32cValue = Mock(return_value=0) + + def _make_path_and_file_list(self, dir_name): + """Utils function for tests.""" + if not os.path.exists(dir_name): + os.mkdir(dir_name) + with open(os.path.join(dir_name, 'summary.001'), 'w'): + pass + + def _make_loader_dict(self, summary_base_dir, dir_num, start_index=0): + """Utils function for making loader dict.""" + if os.path.exists(summary_base_dir): + shutil.rmtree(summary_base_dir) + os.mkdir(summary_base_dir) + loader_dict = dict() + for i in range(dir_num): + log_dir = os.path.join(summary_base_dir, f'job{start_index+i}') + os.mkdir(log_dir) + data_loader = DataLoader(log_dir) + loader_id = log_dir.replace(summary_base_dir, ".") + loader = LoaderStruct(loader_id=loader_id, + name=loader_id, + path=log_dir, + latest_update_time=time.time() + i, + data_loader=data_loader) + loader_dict.update({loader_id: loader}) + return loader_dict + + def test_start_load_data_success(self): + """Test start_load_data method success.""" + summary_base_dir = tempfile.mkdtemp() + train_ids = [] + for i in range(3): + log_path = os.path.join(summary_base_dir, f'dir{i}') + self._make_path_and_file_list(log_path) + train_ids.append(f'./dir{i}') + + data_manager.logger = MockLogger + mock_manager = data_manager.DataManager([DataLoaderGenerator(summary_base_dir)]) + mock_manager.start_load_data(reload_interval=0) + + check_loading_done(mock_manager) + + assert MockLogger.log_msg['info'] == "Load event data end, status: 'DONE', " \ + "and loader pool size is '3'." + shutil.rmtree(summary_base_dir) + + @pytest.mark.parametrize('params', + [{'reload_interval': '30'}, + {'reload_interval': -1}, + {'reload_interval': 30, 'max_threads_count': '20'}, + {'reload_interval': 30, 'max_threads_count': 0}]) + def test_start_load_data_with_invalid_params(self, params): + """Test start_load_data with invalid reload_interval or invalid max_threads_count.""" + summary_base_dir = tempfile.mkdtemp() + d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) + with pytest.raises(ParamValueError): + d_manager.start_load_data(**params) + shutil.rmtree(summary_base_dir) + + def test_list_tensors_success(self): + """Test list_tensors method success.""" + summary_base_dir = tempfile.mkdtemp() + train_job_01 = 'train_01' + name_01 = 'train_job_01' + log_path_01 = os.path.join(summary_base_dir, 'dir1') + self._make_path_and_file_list(log_path_01) + modify_time_01 = 1575460551.9777446 + loader_01 = DataLoader(log_path_01) + + ms_loader = MSDataLoader(log_path_01) + event_data = EventsData() + mock_obj = mock.MagicMock() + mock_obj.samples.return_value = {'test result'} + tag = 'image' + event_data._reservoir_by_tag = {tag: mock_obj} + ms_loader._events_data = event_data + loader_01._loader = ms_loader + + loader = LoaderStruct(loader_id=train_job_01, + name=name_01, + path=log_path_01, + latest_update_time=modify_time_01, + data_loader=loader_01) + loader_pool = {train_job_01: loader} + d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) + d_manager._status = DataManagerStatus.LOADING.value + d_manager._loader_pool = loader_pool + + res = d_manager.list_tensors(train_job_01, tag) + assert res == {'test result'} + + shutil.rmtree(summary_base_dir) + + def test_list_tensors_with_keyerror(self): + """Test list_tensors method with parameter tag raises keyerror.""" + summary_base_dir = tempfile.mkdtemp() + train_job_01 = 'train_01' + name_01 = 'train_job_01' + log_path_01 = os.path.join(summary_base_dir, 'dir1') + self._make_path_and_file_list(log_path_01) + modify_time_01 = 1575460551.9777446 + ms_loader = MSDataLoader(log_path_01) + loader_01 = DataLoader(log_path_01) + loader_01._loader = ms_loader + + loader = LoaderStruct(loader_id=train_job_01, + name=name_01, + path=log_path_01, + latest_update_time=modify_time_01, + data_loader=loader_01) + loader_pool = {train_job_01: loader} + d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) + d_manager._status = DataManagerStatus.LOADING.value + d_manager._loader_pool = loader_pool + tag = 'image' + with pytest.raises(ParamValueError): + d_manager.list_tensors(train_job_01, tag) + + shutil.rmtree(summary_base_dir) + + def test_list_tensors_with_not_exist_train_job(self): + """Test list_tensors method with parameter train_id not found in loader_pool.""" + summary_base_dir = tempfile.mkdtemp() + d_manager = DataManager([DataLoaderGenerator(summary_base_dir)]) + d_manager._status = DataManagerStatus.LOADING.value + tag = 'image' + train_job_01 = 'train_01' + with pytest.raises(ParamValueError): + d_manager.list_tensors(train_job_01, tag) + shutil.rmtree(summary_base_dir) + + @patch.object(data_manager.DataLoaderGenerator, "generate_loaders") + def test_caching(self, mock_generate_loaders): + """Test caching.""" + # Load summaries the first time. + job_num = 10 + summary_base_dir = tempfile.NamedTemporaryFile().name + loader_dict = self._make_loader_dict(summary_base_dir, job_num) + expected_loader_ids = list(loader_dict.keys()) + + mock_generate_loaders.return_value = loader_dict + generators = [data_manager.DataLoaderGenerator(summary_base_dir)] + mock_data_manager = data_manager.DataManager(generators) + mock_data_manager._execute_load_data = Mock() + + mock_data_manager.start_load_data(reload_interval=0) + check_loading_done(mock_data_manager, 3) + current_loader_ids = mock_data_manager._loader_pool.keys() + + assert sorted(current_loader_ids) == sorted(expected_loader_ids) + + # Add new summaries. + new_loader_dict = self._make_loader_dict(summary_base_dir, 6, job_num) + loader_dict.update(new_loader_dict) + expected_loader_ids.extend(list(loader_dict.keys())) + expected_loader_ids = expected_loader_ids[-MAX_DATA_LOADER_SIZE:] + + # Make sure to finish loading, make it init. + mock_data_manager._status = DataManagerStatus.INIT + mock_generate_loaders.return_value = loader_dict + mock_data_manager.start_load_data(reload_interval=0) + check_loading_done(mock_data_manager) + current_loader_ids = mock_data_manager._loader_pool.keys() + + assert sorted(current_loader_ids) == sorted(expected_loader_ids) + + shutil.rmtree(summary_base_dir) diff --git a/tests/ut/datavisual/data_transform/test_events_data.py b/tests/ut/datavisual/data_transform/test_events_data.py new file mode 100644 index 00000000..cce0a57c --- /dev/null +++ b/tests/ut/datavisual/data_transform/test_events_data.py @@ -0,0 +1,116 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.events_data. +Usage: + pytest tests/ut/datavisual +""" +import threading +from collections import namedtuple + +import pytest +from tests.ut.datavisual.mock import MockLogger + +from mindinsight.conf import settings +from mindinsight.datavisual.data_transform import events_data +from mindinsight.datavisual.data_transform.events_data import EventsData, TensorEvent, _Tensor + + +class MockReservoir: + """Use this class to replace reservoir.Reservoir in test.""" + + def __init__(self, size): + self.size = size + self._samples = [_Tensor('wall_time1', 1, 'value1'), _Tensor('wall_time2', 2, 'value2'), + _Tensor('wall_time3', 3, 'value3')] + + def samples(self): + """Replace the samples function.""" + + return self._samples + + def add_sample(self, sample): + """Replace the add_sample function.""" + + self._samples.append(sample) + + def remove_sample(self, sample): + """Replace the remove_sample function.""" + + self._samples.remove(sample) + + +# use for test parameters when tensor event does not have the required attrs +Event1 = namedtuple('tensor_event', 'EVENT_TAG VALUE') + + +class TestEventsData: + """Test EventsData class.""" + + def setup_method(self): + """Mock original logger, init a EventsData object for use.""" + self._ev_data = EventsData() + self._ev_data._tags_by_plugin = {'plugin_name1': [f'tag{i}' for i in range(10)], + 'plugin_name2': [f'tag{i}' for i in range(20, 30)]} + self._ev_data._tags_by_plugin_mutex_lock.update({'plugin_name1': threading.Lock()}) + self._ev_data._reservoir_by_tag = {'tag0': MockReservoir(500), + 'new_tag': MockReservoir(500)} + self._ev_data._tags = [f'tag{i}' for i in range(settings.MAX_TAG_SIZE_PER_EVENTS_DATA)] + + def get_ev_data(self): + """Get the EventsData object.""" + return self._ev_data + + def test_get_tags_by_plugin_name_with_not_exist_key(self): + """Test get_tags_by_plugin_name method when key not exist.""" + + ev_data = self.get_ev_data() + with pytest.raises(KeyError): + ev_data.list_tags_by_plugin('plugin_name3') + + def test_get_tags_by_plugin_name_success(self): + """Test get_tags_by_plugin_name method success.""" + + ev_data = self.get_ev_data() + res = ev_data.list_tags_by_plugin('plugin_name1') + assert set(res) == set(f'tag{i}' for i in range(10)) + + @pytest.mark.parametrize('t_event', [Event1(1, 2)]) + def test_add_tensor_event_with_not_events_data(self, t_event): + """Test when given event do not have attrs tag or value.""" + events_data.logger = MockLogger + ev_data = self.get_ev_data() + + with pytest.raises(TypeError) as ex: + ev_data.add_tensor_event(t_event) + assert ex.value.args[0] == 'Expect to get data of type `TensorEvent`.' + + def test_add_tensor_event_success(self): + """Test add_tensor_event success.""" + + ev_data = self.get_ev_data() + t_event = TensorEvent(wall_time=1, step=4, tag='new_tag', plugin_name='plugin_name1', + value='value1') + + ev_data.add_tensor_event(t_event) + assert 'tag0' not in ev_data._tags + assert ev_data._tags[-1] == 'new_tag' + assert 'tag0' not in ev_data._tags_by_plugin['plugin_name1'] + assert 'tag0' not in ev_data._reservoir_by_tag + assert 'new_tag' in ev_data._tags_by_plugin['plugin_name1'] + assert ev_data._reservoir_by_tag['new_tag'].samples()[-1] == _Tensor(t_event.wall_time, + t_event.step, + t_event.value) diff --git a/tests/ut/datavisual/data_transform/test_ms_data_loader.py b/tests/ut/datavisual/data_transform/test_ms_data_loader.py new file mode 100644 index 00000000..45c9410a --- /dev/null +++ b/tests/ut/datavisual/data_transform/test_ms_data_loader.py @@ -0,0 +1,119 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.ms_data_loader. +Usage: + pytest tests/ut/datavisual +""" +import os +import tempfile +import shutil +from unittest.mock import Mock + +import pytest +from tests.ut.datavisual.mock import MockLogger + +from mindinsight.datavisual.data_transform import ms_data_loader +from mindinsight.datavisual.data_transform.ms_data_loader import MSDataLoader + +# bytes of 3 scalar events +SCALAR_RECORD = (b'\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x96\xe1\xeb)>}\xd7A\x10\x01*' + b'\x11\n\x0f\n\x08tag_name\x1d\r\x06V>\x00\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\t\x96\xe1\xeb)>}\xd7A\x10\x03*\x11\n\x0f\n\x08tag_name\x1d' + b'\xac`\x85>\x00\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x96' + b'\xe1\xeb)>}\xd7A\x10\x05*\x11\n\x0f\n\x08tag_name\x1d\xf80y?\x00\x00\x00\x00') +RECORD_LEN = len(SCALAR_RECORD) + + +class TestMsDataLoader: + """Test ms_data_loader.""" + + @classmethod + def setup_method(cls): + """ + Set up for testing. + Generate a temp directory to store log files and mock the EventsData.EventsData. + """ + mock_exception = Mock(return_value=True) + MockLogger.exception = mock_exception + ms_data_loader.logger = MockLogger + + @pytest.fixture(scope="function") + def crc_pass(self): + """Mock the crc to pass the check.""" + ms_data_loader.crc32.GetValueFromStr = Mock(return_value=0) + ms_data_loader.crc32.GetMaskCrc32cValue = Mock(return_value=0) + + @pytest.fixture(scope="function") + def crc_fail(self): + """Mock the crc to fail the check.""" + ms_data_loader.crc32.GetValueFromStr = Mock(return_value=0) + ms_data_loader.crc32.GetMaskCrc32cValue = Mock(return_value=1) + + def test_check_files_update_success_deleted_files(self): + """Test new file list delete some files.""" + old_file_list = ['summary.01', 'summary.02'] + new_file_list = ['summary02'] + summary_dir = tempfile.mkdtemp() + ms_loader = MSDataLoader(summary_dir) + ms_loader._check_files_deleted(new_file_list, old_file_list) + assert MockLogger.log_msg['warning'] == "There are some files has been deleted, " \ + "we will reload all files in path {}.".format(summary_dir) + shutil.rmtree(summary_dir) + + def test_load_success_with_crc_pass(self, crc_pass): + """Test load success.""" + summary_dir = tempfile.mkdtemp() + file1 = os.path.join(summary_dir, 'summary.01') + write_file(file1, SCALAR_RECORD) + ms_loader = MSDataLoader(summary_dir) + ms_loader._latest_summary_filename = 'summary.00' + ms_loader.load() + assert ms_loader._latest_summary_filename == 'summary.01' + assert ms_loader._latest_summary_file_size == RECORD_LEN + tag = ms_loader.get_events_data().list_tags_by_plugin('scalar') + tensors = ms_loader.get_events_data().tensors(tag[0]) + assert len(tensors) == 3 + + def test_load_with_crc_fail(self, crc_fail): + """Test when crc_fail and will not go to func _event_parse.""" + summary_dir = tempfile.mkdtemp() + file2 = os.path.join(summary_dir, 'summary.02') + write_file(file2, SCALAR_RECORD) + ms_loader = MSDataLoader(summary_dir) + ms_loader.load() + assert 'Check crc faild and ignore this file' in str(MockLogger.log_msg['warning']) + shutil.rmtree(summary_dir) + + def test_filter_event_files(self): + """Test filter_event_files function ok.""" + file_list = ['abc.summary', '123sumary0009abc', 'summary1234', 'aaasummary.5678', + 'summary.0012', 'hellosummary.98786', 'mysummary.123abce', 'summay.4567'] + summary_dir = tempfile.mkdtemp() + for file in file_list: + with open(os.path.join(summary_dir, file), 'w'): + pass + ms_loader = MSDataLoader(summary_dir) + res = ms_loader.filter_valid_files() + expected = sorted(['aaasummary.5678', 'summary.0012', 'hellosummary.98786', 'mysummary.123abce']) + assert sorted(res) == expected + + shutil.rmtree(summary_dir) + +def write_file(filename, record): + """Write bytes strings to file.""" + with open(filename, 'wb') as file: + file.write(record) diff --git a/tests/ut/datavisual/data_transform/test_summary_watcher.py b/tests/ut/datavisual/data_transform/test_summary_watcher.py new file mode 100644 index 00000000..00acb847 --- /dev/null +++ b/tests/ut/datavisual/data_transform/test_summary_watcher.py @@ -0,0 +1,117 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test mindinsight.datavisual.data_transform.summary_watcher. +Usage: + pytest tests/ut/datavisual +""" +import datetime +import math +import os +import random +import shutil +import tempfile + +from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher + + +def gen_directories_and_files(summary_base_dir, file_count, directory_count): + """Generate directories and files for test.""" + end_time = datetime.datetime.now() + start_time = end_time - datetime.timedelta(days=10) + + start_ts = int(start_time.timestamp()) + end_ts = int(end_time.timestamp()) + for _ in range(file_count): + summary = os.path.join(summary_base_dir, f'prefix.summary.{random.randint(start_ts, end_ts)}.suffix') + with open(summary, 'w'): + pass + + os.mkdir(os.path.join(summary_base_dir, 'run')) + for _ in range(file_count): + summary = os.path.join(summary_base_dir, + 'run', + f'prefix.summary.{random.randint(start_ts, end_ts)}.suffix') + with open(summary, 'w'): + pass + + for index in range(directory_count-1): + shutil.copytree(os.path.join(summary_base_dir, 'run'), os.path.join(summary_base_dir, f'run{index}')) + + +class TestSummaryWatcher: + """Test summary watcher.""" + + def test_list_summary_directories_with_overall_on(self): + """Test list_summary_directories method success.""" + summary_base_dir = tempfile.mkdtemp() + file_count = 10 + directory_count = 10 + gen_directories_and_files(summary_base_dir, file_count, directory_count) + + summary_watcher = SummaryWatcher() + directories = summary_watcher.list_summary_directories(summary_base_dir, overall=True) + expected_directory_count = directory_count + 1 + assert len(directories) == min(expected_directory_count, SummaryWatcher.MAX_SUMMARY_DIR_COUNT) + shutil.rmtree(summary_base_dir) + + def test_list_summary_directories_by_pagination(self): + """Test list_summary_directories method success.""" + summary_base_dir = tempfile.mkdtemp() + file_count = 10 + directory_count = 10 + gen_directories_and_files(summary_base_dir, file_count, directory_count) + + summary_watcher = SummaryWatcher() + total, directories = summary_watcher.list_summary_directories_by_pagination( + summary_base_dir, offset=0, limit=10) + + if (file_count + 1) * directory_count + file_count >= SummaryWatcher.MAX_SCAN_COUNT: + expected_directory_count = math.ceil((SummaryWatcher.MAX_SCAN_COUNT - file_count) / (file_count + 1) + 1) + assert total == len(directories) == expected_directory_count + else: + expected_directory_count = directory_count + 1 + assert total == min(expected_directory_count, SummaryWatcher.MAX_SUMMARY_DIR_COUNT) + + shutil.rmtree(summary_base_dir) + + def test_is_summary_directory(self): + """Test is_summary_directory method success.""" + summary_base_dir = tempfile.mkdtemp() + file_count = 1 + directory_count = 1 + gen_directories_and_files(summary_base_dir, file_count, directory_count) + + summary_watcher = SummaryWatcher() + flag = summary_watcher.is_summary_directory(summary_base_dir, './') + assert flag + flag = summary_watcher.is_summary_directory(summary_base_dir, './\x00') + assert not flag + shutil.rmtree(summary_base_dir) + + def test_list_summaries(self): + """Test list_summaries method success.""" + summary_base_dir = tempfile.mkdtemp() + file_count = 10 + directory_count = 1 + gen_directories_and_files(summary_base_dir, file_count, directory_count) + + summary_watcher = SummaryWatcher() + summaries = summary_watcher.list_summaries(summary_base_dir) + assert len(summaries) == file_count + summaries = summary_watcher.list_summaries(summary_base_dir, './\x00') + assert not summaries + shutil.rmtree(summary_base_dir) diff --git a/tests/ut/datavisual/mock.py b/tests/ut/datavisual/mock.py new file mode 100644 index 00000000..828144e0 --- /dev/null +++ b/tests/ut/datavisual/mock.py @@ -0,0 +1,42 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Mock for ut test case. +""" + + +class MockLogger: + """Mock logger in DataLoader and collect log message for verification.""" + log_msg = {'error': None, 'warning': None, 'info': None} + + @classmethod + def error(cls, msg, *args): + """Mock logger.error() and collect error message.""" + cls.log_msg['error'] = msg.replace("%s", "{}").replace("%r", "'{}'").format(*args) + + @classmethod + def warning(cls, msg, *args): + """Mock logger.warning() and collect warning message.""" + cls.log_msg['warning'] = msg.replace("%s", "{}").replace("%r", "'{}'").format(*args) + + @classmethod + def info(cls, msg, *args): + """Mock logger.info() and collect info message.""" + cls.log_msg['info'] = msg.replace("%s", "{}").replace("%r", "'{}'").format(*args) + + @classmethod + def debug(cls, msg, *args): + """Mock logger.debug() and collect debug message.""" + cls.log_msg['debug'] = msg.replace("%s", "{}").replace("%r", "'{}'").format(*args) diff --git a/tests/ut/datavisual/processors/__init__.py b/tests/ut/datavisual/processors/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/processors/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results1.json b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results1.json new file mode 100644 index 00000000..2b4c931a --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results1.json @@ -0,0 +1,16 @@ +{ + "nodes" : [ + { + "attr" : {}, + "input" : {}, + "name" : "Default", + "output" : {}, + "output_i" : -1, + "polymeric_input" : {}, + "polymeric_output" : {}, + "polymeric_scope_name" : "", + "subnode_count" : 3, + "type" : "name_scope" + } + ] +} diff --git a/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results2.json b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results2.json new file mode 100644 index 00000000..6ccfad33 --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results2.json @@ -0,0 +1,20 @@ +{ + "nodes" : [ + { + "attr" : + { + "output_names" : "dtype: DT_GRAPHS\nvalues {\n dtype: DT_FLOAT64\n str_val: \"output\"\n}\n", + "pad_mode" : "dtype: DT_FLOAT64\nstr_val: \"same\"\n" + }, + "input" : {}, + "name" : "Default/conv1-Conv2d/Conv2D1", + "output" : {}, + "output_i" : 0, + "polymeric_input" : {}, + "polymeric_output" : {}, + "polymeric_scope_name" : "", + "subnode_count" : 0, + "type" : "Conv2D" + } + ] +} \ No newline at end of file diff --git a/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results3.json b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results3.json new file mode 100644 index 00000000..5fa3b784 --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_get_nodes_success_expected_results3.json @@ -0,0 +1,437 @@ + +{ + "nodes" : [ + { + "attr" : {}, + "input" : + { + "Default/bn1/Add50" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 1024, 14, 14] + }, + "Default/bn1/conv1.weight" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [64, 3, 7, 7] + }, + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x1" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x10" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x2" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x3" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x4" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x5" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x6" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x7" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x8" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + }, + "Default/bn1/x9" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape1", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/Add50" : + { + "edge_type" : "data" + }, + "Default/bn1/conv1.weight" : + { + "edge_type" : "data" + }, + "Default/bn1/x" : + { + "edge_type" : "data" + }, + "Default/bn1/x1" : + { + "edge_type" : "data" + }, + "Default/bn1/x10" : + { + "edge_type" : "data" + }, + "Default/bn1/x2" : + { + "edge_type" : "data" + }, + "Default/bn1/x3" : + { + "edge_type" : "data" + }, + "Default/bn1/x4" : + { + "edge_type" : "data" + }, + "Default/bn1/x5" : + { + "edge_type" : "data" + }, + "Default/bn1/x6" : + { + "edge_type" : "data" + }, + "Default/bn1/x7" : + { + "edge_type" : "data" + }, + "Default/bn1/x8" : + { + "edge_type" : "data" + }, + "Default/bn1/x9" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/Add51" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 1024, 14, 14] + } + }, + "name" : "Default/bn1/Reshape2", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/Add51" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/Add52" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 1024, 14, 14] + } + }, + "name" : "Default/bn1/Reshape3", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/Add52" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/Add53" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 1024, 14, 14] + } + }, + "name" : "Default/bn1/Reshape4", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/Add53" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/Add54" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 1024, 14, 14] + } + }, + "name" : "Default/bn1/Reshape5", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/Add54" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape6", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape7", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape8", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape9", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape10", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape11", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + }, + { + "attr" : {}, + "input" : + { + "Default/bn1/x" : + { + "edge_type" : "data", + "scope" : "polymeric_scope", + "shape" : [1, 3, 224, 224] + } + }, + "name" : "Default/bn1/Reshape12", + "output" : {}, + "output_i" : 0, + "polymeric_input" : + { + "Default/bn1/x" : + { + "edge_type" : "data" + } + }, + "polymeric_output" : {}, + "polymeric_scope_name" : "Default/bn1/Reshape_1_[12]", + "subnode_count" : 0, + "type" : "Reshape" + } + ] +} diff --git a/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_offset_expected_results1.json b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_offset_expected_results1.json new file mode 100644 index 00000000..b5714405 --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_offset_expected_results1.json @@ -0,0 +1 @@ +{"names":["Default/bn1-BatchNorm2d/cst25","Default/bn1-BatchNorm2d/tuple_getitem105","Default/bn1-BatchNorm2d/tuple_getitem53"]} \ No newline at end of file diff --git a/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results1.json b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results1.json new file mode 100644 index 00000000..4a7931d1 --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results1.json @@ -0,0 +1,4 @@ + +{ + "names" : ["Default", "Default/bn1", "Default/bn1-BatchNorm2d", "Default/bn1-BatchNorm2d/conv1.weight", "Default/bn1-BatchNorm2d/cst25", "Default/bn1-BatchNorm2d/tuple_getitem105", "Default/bn1-BatchNorm2d/tuple_getitem53", "Default/bn1-BatchNorm2d/x11", "Default/bn1-BatchNorm2d/x12", "Default/bn1-BatchNorm2d/x13", "Default/bn1-BatchNorm2d/x14", "Default/bn1-BatchNorm2d/x15", "Default/bn1-BatchNorm2d/x16", "Default/bn1-BatchNorm2d/x17", "Default/bn1-BatchNorm2d/x18", "Default/bn1-BatchNorm2d/x19", "Default/bn1-BatchNorm2d/x20", "Default/bn1/Add50", "Default/bn1/Add51", "Default/bn1/Add52", "Default/bn1/Add53", "Default/bn1/Add54", "Default/bn1/Reshape1", "Default/bn1/Reshape10", "Default/bn1/Reshape11", "Default/bn1/Reshape12", "Default/bn1/Reshape2", "Default/bn1/Reshape3", "Default/bn1/Reshape4", "Default/bn1/Reshape5", "Default/bn1/Reshape6", "Default/bn1/Reshape7", "Default/bn1/Reshape8", "Default/bn1/Reshape9", "Default/bn1/Reshape_1_[12]", "Default/bn1/conv1.weight", "Default/bn1/cst13", "Default/bn1/x", "Default/bn1/x1", "Default/bn1/x10", "Default/bn1/x11", "Default/bn1/x2", "Default/bn1/x3", "Default/bn1/x4", "Default/bn1/x5", "Default/bn1/x6", "Default/bn1/x7", "Default/bn1/x8", "Default/bn1/x9", "Default/conv1-Conv2d", "Default/conv1-Conv2d/Conv2D1"] +} \ No newline at end of file diff --git a/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results2.json b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results2.json new file mode 100644 index 00000000..b98e912a --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_search_node_names_with_search_content_expected_results2.json @@ -0,0 +1 @@ +{"names":["Default/bn1","Default/bn1-BatchNorm2d","Default/bn1-BatchNorm2d/conv1.weight","Default/bn1-BatchNorm2d/cst25","Default/bn1-BatchNorm2d/tuple_getitem105","Default/bn1-BatchNorm2d/tuple_getitem53","Default/bn1-BatchNorm2d/x11","Default/bn1-BatchNorm2d/x12","Default/bn1-BatchNorm2d/x13","Default/bn1-BatchNorm2d/x14","Default/bn1-BatchNorm2d/x15","Default/bn1-BatchNorm2d/x16","Default/bn1-BatchNorm2d/x17","Default/bn1-BatchNorm2d/x18","Default/bn1-BatchNorm2d/x19","Default/bn1-BatchNorm2d/x20","Default/bn1/Add50","Default/bn1/Add51","Default/bn1/Add52","Default/bn1/Add53","Default/bn1/Add54","Default/bn1/Reshape1","Default/bn1/Reshape10","Default/bn1/Reshape11","Default/bn1/Reshape12","Default/bn1/Reshape2","Default/bn1/Reshape3","Default/bn1/Reshape4","Default/bn1/Reshape5","Default/bn1/Reshape6","Default/bn1/Reshape7","Default/bn1/Reshape8","Default/bn1/Reshape9","Default/bn1/Reshape_1_[12]","Default/bn1/conv1.weight","Default/bn1/cst13","Default/bn1/x","Default/bn1/x1","Default/bn1/x10","Default/bn1/x11","Default/bn1/x2","Default/bn1/x3","Default/bn1/x4","Default/bn1/x5","Default/bn1/x6","Default/bn1/x7","Default/bn1/x8","Default/bn1/x9"]} diff --git a/tests/ut/datavisual/processors/graph_results/test_search_single_node_success_expected_results1.json b/tests/ut/datavisual/processors/graph_results/test_search_single_node_success_expected_results1.json new file mode 100644 index 00000000..206cd26c --- /dev/null +++ b/tests/ut/datavisual/processors/graph_results/test_search_single_node_success_expected_results1.json @@ -0,0 +1 @@ +{"children":{"children":{},"nodes":[{"attr":{},"input":{},"name":"Default/conv1-Conv2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":1,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1-BatchNorm2d","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":14,"type":"name_scope"},{"attr":{},"input":{},"name":"Default/bn1","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":20,"type":"name_scope"}],"scope_name":"Default"},"nodes":[{"attr":{},"input":{},"name":"Default","output":{},"output_i":-1,"polymeric_input":{},"polymeric_output":{},"polymeric_scope_name":"","subnode_count":3,"type":"name_scope"}],"scope_name":""} diff --git a/tests/ut/datavisual/processors/test_graph_processor.py b/tests/ut/datavisual/processors/test_graph_processor.py new file mode 100644 index 00000000..87730ed4 --- /dev/null +++ b/tests/ut/datavisual/processors/test_graph_processor.py @@ -0,0 +1,250 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test graph processor. +Usage: + pytest tests/ut/datavisual +""" +import os +import json +import tempfile + +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.log_operations import LogOperations +from tests.ut.datavisual.utils.utils import check_loading_done, delete_files_or_dirs + +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform import data_manager +from mindinsight.datavisual.data_transform.data_manager import DataManager +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.datavisual.processors.graph_processor import GraphProcessor +from mindinsight.datavisual.utils import crc32 +from mindinsight.utils.exceptions import ParamValueError + + +class TestGraphProcessor: + """Test Graph Processor api.""" + _steps_list = [1, 3, 5] + + _temp_path = None + _graph_dict = None + _mock_data_manager = None + _train_id = None + + _generated_path = [] + + graph_results_dir = os.path.join(os.path.dirname(__file__), 'graph_results') + + @classmethod + def setup_class(cls): + """Mock common environment for graph unittest.""" + crc32.GetValueFromStr = Mock(return_value=0) + crc32.GetMaskCrc32cValue = Mock(return_value=0) + data_manager.logger = MockLogger + + def teardown_class(self): + """Delete temp files.""" + delete_files_or_dirs(self._generated_path) + + @pytest.fixture(scope='function') + def load_graph_record(self): + """Load graph record.""" + summary_base_dir = tempfile.mkdtemp() + log_dir = tempfile.mkdtemp(dir=summary_base_dir) + + self._train_id = log_dir.replace(summary_base_dir, ".") + + graph_base_path = os.path.join(os.path.dirname(__file__), + os.pardir, "utils", "log_generators", "graph_base.json") + self._temp_path, self._graph_dict = LogOperations.generate_log( + PluginNameEnum.GRAPH.value, log_dir, dict(graph_base_path=graph_base_path)) + + self._generated_path.append(summary_base_dir) + + self._mock_data_manager = data_manager.DataManager( + [DataLoaderGenerator(summary_base_dir)]) + self._mock_data_manager.start_load_data(reload_interval=0) + + # wait for loading done + check_loading_done(self._mock_data_manager, time_limit=5) + + @pytest.fixture(scope='function') + def load_no_graph_record(self): + """Load no graph record.""" + summary_base_dir = tempfile.mkdtemp() + log_dir = tempfile.mkdtemp(dir=summary_base_dir) + self._train_id = log_dir.replace(summary_base_dir, ".") + + self._temp_path, _, _ = LogOperations.generate_log( + PluginNameEnum.IMAGE.value, log_dir, dict(steps=self._steps_list, tag="image")) + + self._generated_path.append(summary_base_dir) + + self._mock_data_manager = data_manager.DataManager( + [DataLoaderGenerator(summary_base_dir)]) + self._mock_data_manager.start_load_data(reload_interval=0) + + # wait for loading done + check_loading_done(self._mock_data_manager, time_limit=5) + + def compare_result_with_file(self, result, filename): + """Compare result with file which contain the expected results.""" + with open(os.path.join(self.graph_results_dir, filename), 'r') as fp: + expected_results = json.load(fp) + assert result == expected_results + + def test_get_nodes_with_not_exist_train_id(self, load_graph_record): + """Test getting nodes with not exist train id.""" + test_train_id = "not_exist_train_id" + with pytest.raises(ParamValueError) as exc_info: + GraphProcessor(test_train_id, self._mock_data_manager) + assert "Can not find the train job in data manager." in exc_info.value.message + + @patch.object(DataManager, 'get_train_job_by_plugin') + def test_get_nodes_with_loader_is_none(self, mock_get_train_job_by_plugin, load_graph_record): + """Test get nodes with loader is None.""" + mock_get_train_job_by_plugin.return_value = None + with pytest.raises(exceptions.SummaryLogPathInvalid): + GraphProcessor(self._train_id, self._mock_data_manager) + + assert mock_get_train_job_by_plugin.called + + @pytest.mark.parametrize("name, node_type", [ + ("not_exist_name", "name_scope"), + ("", "polymeric_scope") + ]) + def test_get_nodes_with_not_exist_name(self, load_graph_record, name, node_type): + """Test getting nodes with not exist name.""" + with pytest.raises(ParamValueError) as exc_info: + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + graph_processor.get_nodes(name, node_type) + + if name: + assert "The node name is not in graph." in exc_info.value.message + else: + assert f'The node name "{name}" not in graph, node type is {node_type}.' in exc_info.value.message + + @pytest.mark.parametrize("name, node_type, result_file", [ + (None, 'name_scope', 'test_get_nodes_success_expected_results1.json'), + ('Default/conv1-Conv2d', 'name_scope', 'test_get_nodes_success_expected_results2.json'), + ('Default/bn1/Reshape_1_[12]', 'polymeric_scope', 'test_get_nodes_success_expected_results3.json') + ]) + def test_get_nodes_success(self, load_graph_record, name, node_type, result_file): + """Test getting nodes successfully.""" + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + results = graph_processor.get_nodes(name, node_type) + self.compare_result_with_file(results, result_file) + + @pytest.mark.parametrize("search_content, result_file", [ + (None, 'test_search_node_names_with_search_content_expected_results1.json'), + ('Default/bn1', 'test_search_node_names_with_search_content_expected_results2.json'), + ('not_exist_search_content', None) + ]) + def test_search_node_names_with_search_content(self, load_graph_record, + search_content, + result_file): + """Test search node names with search content.""" + test_offset = 0 + test_limit = 1000 + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + results = graph_processor.search_node_names(search_content, + test_offset, + test_limit) + if search_content == 'not_exist_search_content': + expected_results = {'names': []} + assert results == expected_results + else: + self.compare_result_with_file(results, result_file) + + @pytest.mark.parametrize("offset", [-100, -1]) + def test_search_node_names_with_negative_offset(self, load_graph_record, offset): + """Test search node names with negative offset.""" + test_search_content = "" + test_limit = 3 + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + with pytest.raises(ParamValueError) as exc_info: + graph_processor.search_node_names(test_search_content, offset, test_limit) + assert "'offset' should be greater than or equal to 0." in exc_info.value.message + + @pytest.mark.parametrize("offset, result_file", [ + (1, 'test_search_node_names_with_offset_expected_results1.json') + ]) + def test_search_node_names_with_offset(self, load_graph_record, offset, result_file): + """Test search node names with offset.""" + test_search_content = "Default/bn1" + test_offset = offset + test_limit = 3 + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + results = graph_processor.search_node_names(test_search_content, + test_offset, + test_limit) + self.compare_result_with_file(results, result_file) + + def test_search_node_names_with_wrong_limit(self, load_graph_record): + """Test search node names with wrong limit.""" + test_search_content = "" + test_offset = 0 + test_limit = 0 + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + with pytest.raises(ParamValueError) as exc_info: + graph_processor.search_node_names(test_search_content, test_offset, + test_limit) + assert "'limit' should in [1, 1000]." in exc_info.value.message + + @pytest.mark.parametrize("name, result_file", [ + ('Default/bn1', 'test_search_single_node_success_expected_results1.json') + ]) + def test_search_single_node_success(self, load_graph_record, name, result_file): + """Test searching single node successfully.""" + + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + results = graph_processor.search_single_node(name) + self.compare_result_with_file(results, result_file) + + + def test_search_single_node_with_not_exist_name(self, load_graph_record): + """Test searching single node with not exist name.""" + test_name = "not_exist_name" + + with pytest.raises(exceptions.NodeNotInGraphError): + graph_processor = GraphProcessor(self._train_id, + self._mock_data_manager) + graph_processor.search_single_node(test_name) + + def test_check_graph_status_no_graph(self, load_no_graph_record): + """Test checking graph status no graph.""" + with pytest.raises(ParamValueError) as exc_info: + GraphProcessor(self._train_id, self._mock_data_manager) + assert exc_info.value.message == "Invalid parameter value. Can not find any graph data " \ + "in the train job." diff --git a/tests/ut/datavisual/processors/test_images_processor.py b/tests/ut/datavisual/processors/test_images_processor.py new file mode 100644 index 00000000..4062f1a3 --- /dev/null +++ b/tests/ut/datavisual/processors/test_images_processor.py @@ -0,0 +1,235 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test images processor. +Usage: + pytest tests/ut/datavisual +""" +import tempfile +from unittest.mock import Mock + +import pytest +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.log_operations import LogOperations +from tests.ut.datavisual.utils.utils import check_loading_done, delete_files_or_dirs + +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform import data_manager +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.datavisual.processors.images_processor import ImageProcessor +from mindinsight.datavisual.utils import crc32 +from mindinsight.utils.exceptions import ParamValueError + + +class TestImagesProcessor: + """Test images processor api.""" + _steps_list = [1, 3, 5, 7, 8] + _more_steps_list = list(range(30)) + _cross_steps_list = [1, 3, 5, 7, 9, 2, 3, 4, 15] + + _tag_name = 'tag_name' + _plugin_name = 'image' + _complete_tag_name = f'{_tag_name}/{_plugin_name}' + + _temp_path = None + _images_values = None + _images_metadata = None + _mock_data_manager = None + _train_id = None + + _generated_path = [] + + @classmethod + def setup_class(cls): + """Mock common environment for images unittest.""" + crc32.GetValueFromStr = Mock(return_value=0) + crc32.GetMaskCrc32cValue = Mock(return_value=0) + data_manager.logger = MockLogger + + def teardown_class(self): + """Delete temp files.""" + delete_files_or_dirs(self._generated_path) + + def _init_data_manager(self, steps_list): + """ + Generate log and init data_manager. + + Args: + steps_list (list): Init steps. + + """ + summary_base_dir = tempfile.mkdtemp() + log_dir = tempfile.mkdtemp(dir=summary_base_dir) + + self._train_id = log_dir.replace(summary_base_dir, ".") + + self._temp_path, self._images_metadata, self._images_values = LogOperations.generate_log( + PluginNameEnum.IMAGE.value, log_dir, dict(steps=steps_list, tag=self._tag_name)) + + self._generated_path.append(summary_base_dir) + + self._mock_data_manager = data_manager.DataManager([DataLoaderGenerator(summary_base_dir)]) + self._mock_data_manager.start_load_data(reload_interval=0) + + # wait for loading done + check_loading_done(self._mock_data_manager, time_limit=5) + + @pytest.fixture(scope='function') + def load_image_record(self): + """Load image record.""" + self._init_data_manager(self._steps_list) + + @pytest.fixture(scope='function') + def load_more_than_limit_image_record(self): + """Load image record.""" + self._init_data_manager(self._more_steps_list) + + @pytest.fixture(scope='function') + def load_reservoir_remove_sample_image_record(self): + """Load image record.""" + self._init_data_manager(self._cross_steps_list) + + def test_get_metadata_list_with_not_exist_id(self, load_image_record): + """Test getting metadata list with not exist id.""" + test_train_id = 'not_exist_id' + image_processor = ImageProcessor(self._mock_data_manager) + with pytest.raises(ParamValueError) as exc_info: + image_processor.get_metadata_list(test_train_id, self._tag_name) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in loader pool about the train job." in exc_info.value.message + + def test_get_metadata_list_with_not_exist_tag(self, load_image_record): + """Test get metadata list with not exist tag.""" + test_tag_name = 'not_exist_tag_name' + + image_processor = ImageProcessor(self._mock_data_manager) + + with pytest.raises(ParamValueError) as exc_info: + image_processor.get_metadata_list(self._train_id, test_tag_name) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in this train job by given tag." in exc_info.value.message + + def test_get_metadata_list_success(self, load_image_record): + """Test getting metadata list success.""" + test_tag_name = self._complete_tag_name + + image_processor = ImageProcessor(self._mock_data_manager) + results = image_processor.get_metadata_list(self._train_id, test_tag_name).get('metadatas') + + assert results == self._images_metadata + + def test_get_single_image_with_not_exist_id(self, load_image_record): + """Test getting single image with not exist id.""" + test_train_id = 'not_exist_id' + test_tag_name = self._complete_tag_name + test_step = self._steps_list[0] + image_processor = ImageProcessor(self._mock_data_manager) + + with pytest.raises(ParamValueError) as exc_info: + image_processor.get_single_image(test_train_id, test_tag_name, test_step) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in loader pool about the train job." in exc_info.value.message + + def test_get_single_image_with_not_exist_tag(self, load_image_record): + """Test getting single image with not exist tag.""" + test_tag_name = 'not_exist_tag_name' + test_step = self._steps_list[0] + + image_processor = ImageProcessor(self._mock_data_manager) + + with pytest.raises(ParamValueError) as exc_info: + image_processor.get_single_image(self._train_id, test_tag_name, test_step) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in this train job by given tag." in exc_info.value.message + + def test_get_single_image_with_not_exist_step(self, load_image_record): + """Test getting single image with not exist step.""" + test_tag_name = self._complete_tag_name + test_step = 10000 + + image_processor = ImageProcessor(self._mock_data_manager) + + with pytest.raises(ParamValueError) as exc_info: + image_processor.get_single_image(self._train_id, test_tag_name, test_step) + + assert exc_info.value.error_code == '50540002' + assert "Can not find the step with given train job id and tag." in exc_info.value.message + + def test_get_single_image_success(self, load_image_record): + """Test getting single image successfully.""" + test_tag_name = self._complete_tag_name + test_step_index = 0 + test_step = self._steps_list[test_step_index] + + image_processor = ImageProcessor(self._mock_data_manager) + + results = image_processor.get_single_image(self._train_id, test_tag_name, test_step) + + expected_image_tensor = self._images_values.get(test_step) + + image_generator = LogOperations.get_log_generator(PluginNameEnum.IMAGE.value) + recv_image_tensor = image_generator.get_image_tensor_from_bytes(results) + + assert recv_image_tensor.any() == expected_image_tensor.any() + + def test_reservoir_add_sample(self, load_more_than_limit_image_record): + """Test adding sample in reservoir.""" + test_tag_name = self._complete_tag_name + + cnt = 0 + + for step in self._more_steps_list: + test_step = step + + image_processor = ImageProcessor(self._mock_data_manager) + + try: + image_processor.get_single_image(self._train_id, test_tag_name, test_step) + except ParamValueError: + cnt += 1 + assert len(self._more_steps_list) - cnt == 10 + + def test_reservoir_remove_sample(self, load_reservoir_remove_sample_image_record): + """ + Test removing sample in reservoir. + + If step list is [1, 3, 5, 7, 9, 2, 3, 4, 15], + and then [3, 5, 7, 9] will be deleted. + Results will be [1, 2, 3, 4, 15]. + """ + test_tag_name = self._complete_tag_name + + not_found_step_list = [] + current_step_list = [] + + steps_list = set(self._cross_steps_list) + for step in steps_list: + test_step = step + + image_processor = ImageProcessor(self._mock_data_manager) + + try: + image_processor.get_single_image(self._train_id, test_tag_name, test_step) + current_step_list.append(test_step) + except ParamValueError: + not_found_step_list.append(test_step) + + assert current_step_list == [1, 2, 3, 4, 15] + assert not_found_step_list == [5, 7, 9] diff --git a/tests/ut/datavisual/processors/test_scalars_processor.py b/tests/ut/datavisual/processors/test_scalars_processor.py new file mode 100644 index 00000000..079192a9 --- /dev/null +++ b/tests/ut/datavisual/processors/test_scalars_processor.py @@ -0,0 +1,116 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test scalars processor. +Usage: + pytest tests/ut/datavisual +""" +import tempfile +from unittest.mock import Mock + +import pytest +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.log_operations import LogOperations +from tests.ut.datavisual.utils.utils import check_loading_done, delete_files_or_dirs + +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform import data_manager +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor +from mindinsight.datavisual.utils import crc32 +from mindinsight.utils.exceptions import ParamValueError + + +class TestScalarsProcessor: + """Test scalar processor api.""" + _steps_list = [1, 3, 5] + _tag_name = 'tag_name' + _plugin_name = 'scalar' + _complete_tag_name = f'{_tag_name}/{_plugin_name}' + + _temp_path = None + _scalars_values = None + _scalars_metadata = None + _mock_data_manager = None + _train_id = None + + _generated_path = [] + + @classmethod + def setup_class(cls): + """Mock common environment for scalars unittest.""" + crc32.GetValueFromStr = Mock(return_value=0) + crc32.GetMaskCrc32cValue = Mock(return_value=0) + data_manager.logger = MockLogger + + def teardown_class(self): + """Delete temp files.""" + delete_files_or_dirs(self._generated_path) + + @pytest.fixture(scope='function') + def load_scalar_record(self): + """Load scalar record.""" + summary_base_dir = tempfile.mkdtemp() + log_dir = tempfile.mkdtemp(dir=summary_base_dir) + + self._train_id = log_dir.replace(summary_base_dir, ".") + + self._temp_path, self._scalars_metadata, self._scalars_values = LogOperations.generate_log( + PluginNameEnum.SCALAR.value, log_dir, dict(step=self._steps_list, tag=self._tag_name)) + + self._generated_path.append(summary_base_dir) + + self._mock_data_manager = data_manager.DataManager([DataLoaderGenerator(summary_base_dir)]) + self._mock_data_manager.start_load_data(reload_interval=0) + + # wait for loading done + check_loading_done(self._mock_data_manager, time_limit=5) + + def test_get_metadata_list_with_not_exist_id(self, load_scalar_record): + """Get metadata list with not exist id.""" + test_train_id = 'not_exist_id' + scalar_processor = ScalarsProcessor(self._mock_data_manager) + with pytest.raises(ParamValueError) as exc_info: + scalar_processor.get_metadata_list(test_train_id, self._tag_name) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in loader pool about the train job." in exc_info.value.message + + def test_get_metadata_list_with_not_exist_tag(self, load_scalar_record): + """Get metadata list with not exist tag.""" + test_tag_name = 'not_exist_tag_name' + + scalar_processor = ScalarsProcessor(self._mock_data_manager) + + with pytest.raises(ParamValueError) as exc_info: + scalar_processor.get_metadata_list(self._train_id, test_tag_name) + + assert exc_info.value.error_code == '50540002' + assert "Can not find any data in this train job by given tag." in exc_info.value.message + + def test_get_metadata_list_success(self, load_scalar_record): + """Get metadata list success.""" + test_tag_name = self._complete_tag_name + + scalar_processor = ScalarsProcessor(self._mock_data_manager) + results = scalar_processor.get_metadata_list(self._train_id, test_tag_name) + + recv_metadata = results.get('metadatas') + + for recv_values, expected_values in zip(recv_metadata, self._scalars_metadata): + assert recv_values.get('wall_time') == expected_values.get('wall_time') + assert recv_values.get('step') == expected_values.get('step') + assert abs(recv_values.get('value') - expected_values.get('value')) < 1e-6 diff --git a/tests/ut/datavisual/processors/test_train_task_manager.py b/tests/ut/datavisual/processors/test_train_task_manager.py new file mode 100644 index 00000000..3ea552b3 --- /dev/null +++ b/tests/ut/datavisual/processors/test_train_task_manager.py @@ -0,0 +1,152 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Function: + Test train task manager. +Usage: + pytest tests/ut/datavisual +""" +import os +import tempfile +import time +from unittest.mock import Mock + +import pytest +from tests.ut.datavisual.mock import MockLogger +from tests.ut.datavisual.utils.log_operations import LogOperations +from tests.ut.datavisual.utils.utils import check_loading_done, delete_files_or_dirs + +from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.data_transform import data_manager +from mindinsight.datavisual.data_transform.loader_generators.data_loader_generator import DataLoaderGenerator +from mindinsight.datavisual.processors.train_task_manager import TrainTaskManager +from mindinsight.datavisual.utils import crc32 +from mindinsight.utils.exceptions import ParamValueError + + +class TestTrainTaskManager: + """Test train task manager.""" + _root_dir = None + + _dir_num = 3 + _plugins_id_map = {'image': [], 'scalar': [], 'graph': []} + _events_names = [] + _steps_list = [1, 3, 5] + _steps = len(_steps_list) + + _tag_name = 'tag_name' + + _mock_data_manager = None + + _train_id_list = [] + + _generated_path = [] + + @classmethod + def setup_class(cls): + """ + Mock common environment for train task unittest. + """ + crc32.GetValueFromStr = Mock(return_value=0) + crc32.GetMaskCrc32cValue = Mock(return_value=0) + data_manager.logger = MockLogger + + def teardown_class(self): + """Delete temp files.""" + delete_files_or_dirs(self._generated_path) + + @pytest.fixture(scope='function') + def load_data(self): + """Load data.""" + self._plugins_id_map = {'image': [], 'scalar': [], 'graph': []} + self._events_names = [] + self._train_id_list = [] + + graph_base_path = os.path.join(os.path.dirname(__file__), + os.pardir, "utils", "log_generators", "graph_base.json") + + self._root_dir = tempfile.mkdtemp() + for i in range(self._dir_num): + dir_path = tempfile.mkdtemp(dir=self._root_dir) + + tmp_tag_name = self._tag_name + '_' + str(i) + event_name = str(i) + "_name" + train_id = dir_path.replace(self._root_dir, ".") + + # Pass timestamp to write to the same file. + log_settings = dict( + steps=self._steps_list, + tag=tmp_tag_name, + graph_base_path=graph_base_path, + time=time.time()) + + if i % 3 != 0: + LogOperations.generate_log(PluginNameEnum.IMAGE.value, dir_path, log_settings) + self._plugins_id_map['image'].append(train_id) + if i % 3 != 1: + LogOperations.generate_log(PluginNameEnum.SCALAR.value, dir_path, log_settings) + self._plugins_id_map['scalar'].append(train_id) + if i % 3 != 2: + LogOperations.generate_log(PluginNameEnum.GRAPH.value, dir_path, log_settings) + self._plugins_id_map['graph'].append(train_id) + self._events_names.append(event_name) + + self._train_id_list.append(train_id) + + self._generated_path.append(self._root_dir) + + self._mock_data_manager = data_manager.DataManager([DataLoaderGenerator(self._root_dir)]) + self._mock_data_manager.start_load_data(reload_interval=0) + + check_loading_done(self._mock_data_manager, time_limit=30) + + def test_get_single_train_task_with_not_exists_train_id(self, load_data): + """Test getting single train task with not exists train_id.""" + train_task_manager = TrainTaskManager(self._mock_data_manager) + for plugin_name in PluginNameEnum.list_members(): + test_train_id = "not_exist_id" + with pytest.raises(ParamValueError) as exc_info: + _ = train_task_manager.get_single_train_task(plugin_name, test_train_id) + assert exc_info.type == ParamValueError + assert exc_info.value.message == "Invalid parameter value. Can not find " \ + "the train job in data manager." + assert exc_info.value.error_code == '50540002' + + def test_get_single_train_task_with_params(self, load_data): + """Test getting single train task with params.""" + train_task_manager = TrainTaskManager(self._mock_data_manager) + for plugin_name in PluginNameEnum.list_members(): + for test_train_id in self._train_id_list: + result = train_task_manager.get_single_train_task(plugin_name, test_train_id) + tags = result.get("train_jobs")[0].get("tags") + + # if it is a UUID + if tags: + assert test_train_id in self._plugins_id_map.get(plugin_name) + else: + assert test_train_id not in self._plugins_id_map.get(plugin_name) + + def test_get_plugins_with_train_id(self, load_data): + """Test getting plugins with train id.""" + train_task_manager = TrainTaskManager(self._mock_data_manager) + + for train_id in self._train_id_list: + result = train_task_manager.get_plugins(train_id) + plugins = result.get('plugins') + for plugin_name in plugins: + if plugins.get(plugin_name): + assert train_id in self._plugins_id_map.get(plugin_name) + else: + assert train_id not in self._plugins_id_map.get(plugin_name) diff --git a/tests/ut/datavisual/utils/__init__.py b/tests/ut/datavisual/utils/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/utils/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/utils/crc32.py b/tests/ut/datavisual/utils/crc32.py new file mode 100644 index 00000000..06c3a00d --- /dev/null +++ b/tests/ut/datavisual/utils/crc32.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mask string to crc32.""" +CRC_TABLE_32 = ( + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, + 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C, + 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, + 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, + 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, + 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, + 0x6EF07595, 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, 0x5125DAD3, + 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F, + 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, 0x61C69362, 0x93AD1061, 0x80FDE395, + 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, + 0x44694011, 0x5739B3E5, 0xA55230E6, 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, + 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, + 0x563C5F93, 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, 0x1871A4D8, + 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5, + 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, + 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, + 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, + 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, + 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, + 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6, + 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, 0xF36E6F75, 0x0105EC76, 0x12551F82, + 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +) + +_CRC = 0 +_MASK = 0xFFFFFFFF + + +def _uint32(x): + """Transform x's type to uint32.""" + return x & 0xFFFFFFFF + + +def _get_crc_checksum(crc, data): + """Get crc checksum.""" + crc ^= _MASK + for d in data: + crc_table_index = (crc ^ d) & 0xFF + crc = (CRC_TABLE_32[crc_table_index] ^ (crc >> 8)) & _MASK + + crc ^= _MASK + + return crc + + +def get_mask_from_string(data): + """ + Get masked crc from data. + + Args: + data (byte): Byte string of data. + + Returns: + uint32, masked crc. + + """ + + crc = _get_crc_checksum(_CRC, data) + crc = _uint32(crc & _MASK) + crc = _uint32(((crc >> 15) | _uint32(crc << 17)) + 0xA282EAD8) + + return crc diff --git a/tests/ut/datavisual/utils/log_generators/__init__.py b/tests/ut/datavisual/utils/log_generators/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/datavisual/utils/log_generators/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/datavisual/utils/log_generators/graph_base.json b/tests/ut/datavisual/utils/log_generators/graph_base.json new file mode 100644 index 00000000..ff4d5416 --- /dev/null +++ b/tests/ut/datavisual/utils/log_generators/graph_base.json @@ -0,0 +1,1414 @@ +{ + "node": [ + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + }, + { + "name": "x1", + "type": "DATA_EDGE" + }, + { + "name": "x2", + "type": "DATA_EDGE" + }, + { + "name": "x3", + "type": "DATA_EDGE" + }, + { + "name": "x4", + "type": "DATA_EDGE" + }, + { + "name": "x5", + "type": "DATA_EDGE" + }, + { + "name": "x6", + "type": "DATA_EDGE" + }, + { + "name": "x7", + "type": "DATA_EDGE" + }, + { + "name": "x8", + "type": "DATA_EDGE" + }, + { + "name": "x9", + "type": "DATA_EDGE" + }, + { + "name": "x10", + "type": "DATA_EDGE" + }, + { + "name": "conv1.weight", + "type": "DATA_EDGE" + } + ], + "name": "1", + "opType": "Conv2D", + "scope": "Default/conv1-Conv2d", + "attribute": [ + { + "name": "output_names", + "value": { + "dtype": "DT_GRAPHS", + "values": [ + { + "dtype": "DT_FLOAT64", + "strVal": "output" + } + ] + } + }, + { + "name": "pad_mode", + "value": { + "dtype": "DT_FLOAT64", + "strVal": "same" + } + } + ], + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "64" + }, + { + "size": "112" + }, + { + "size": "112" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + }, + { + "name": "x1", + "type": "DATA_EDGE" + }, + { + "name": "x2", + "type": "DATA_EDGE" + }, + { + "name": "x3", + "type": "DATA_EDGE" + }, + { + "name": "x4", + "type": "DATA_EDGE" + }, + { + "name": "x5", + "type": "DATA_EDGE" + }, + { + "name": "x6", + "type": "DATA_EDGE" + }, + { + "name": "x7", + "type": "DATA_EDGE" + }, + { + "name": "x8", + "type": "DATA_EDGE" + }, + { + "name": "x9", + "type": "DATA_EDGE" + }, + { + "name": "x10", + "type": "DATA_EDGE" + }, + { + "name": "cst13", + "type": "DATA_EDGE" + } + ], + "name": "53", + "opType": "tuple_getitem", + "scope": "Default/bn1-BatchNorm2d", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + }, + { + "name": "x12", + "type": "DATA_EDGE" + }, + { + "name": "x13", + "type": "DATA_EDGE" + }, + { + "name": "x14", + "type": "DATA_EDGE" + }, + { + "name": "x15", + "type": "DATA_EDGE" + }, + { + "name": "x16", + "type": "DATA_EDGE" + }, + { + "name": "x17", + "type": "DATA_EDGE" + }, + { + "name": "x18", + "type": "DATA_EDGE" + }, + { + "name": "x19", + "type": "DATA_EDGE" + }, + { + "name": "x20", + "type": "DATA_EDGE" + }, + { + "name": "conv1.weight", + "type": "DATA_EDGE" + }, + { + "name": "cst25", + "type": "DATA_EDGE" + } + ], + "name": "105", + "opType": "tuple_getitem", + "scope": "Default/bn1-BatchNorm2d", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "50", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "51", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "52", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "53", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x11", + "type": "DATA_EDGE" + } + ], + "name": "54", + "opType": "Add", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "1024" + }, + { + "size": "14" + }, + { + "size": "14" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "50", + "type": "DATA_EDGE" + } + ], + "name": "1", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "51", + "type": "DATA_EDGE" + } + ], + "name": "2", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "52", + "type": "DATA_EDGE" + } + ], + "name": "3", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "53", + "type": "DATA_EDGE" + } + ], + "name": "4", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "54", + "type": "DATA_EDGE" + } + ], + "name": "5", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "6", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "7", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "8", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "9", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "10", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "11", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + }, + { + "input": [ + { + "name": "x", + "type": "DATA_EDGE" + } + ], + "name": "12", + "opType": "Reshape", + "scope": "Default/bn1", + "outputType": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "128" + }, + { + "size": "28" + }, + { + "size": "28" + } + ] + } + } + } + } + ], + "name": "849_848_847_424_1_construct", + "parameters": [ + { + "name": "x", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "conv1.weight", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "64" + }, + { + "size": "3" + }, + { + "size": "7" + }, + { + "size": "7" + } + ] + } + } + } + }, + { + "name": "x1", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x2", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x3", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x4", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x5", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x6", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x7", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x8", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x9", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x10", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x11", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x12", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x13", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x14", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x15", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x16", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x17", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x18", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x19", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + }, + { + "name": "x20", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "3" + }, + { + "size": "224" + }, + { + "size": "224" + } + ] + } + } + } + } + ], + "outputs": [ + { + "name": "228", + "type": { + "dataType": "DT_STRING", + "tensorType": { + "elemType": "DT_FLOAT16", + "shape": { + "dim": [ + { + "size": "1" + }, + { + "size": "10" + } + ] + } + } + } + } + ], + "constVals": [ + { + "key": "cst12", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + }, + { + "key": "cst13", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + }, + { + "key": "cst25", + "value": { + "dtype": "DT_INT32", + "intVal": "0" + } + } + ] +} diff --git a/tests/ut/datavisual/utils/log_generators/graph_log_generator.py b/tests/ut/datavisual/utils/log_generators/graph_log_generator.py new file mode 100644 index 00000000..3cd85832 --- /dev/null +++ b/tests/ut/datavisual/utils/log_generators/graph_log_generator.py @@ -0,0 +1,80 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log generator for graph.""" +import json +import os +import time + +from google.protobuf import json_format + +from tests.ut.datavisual.utils.log_generators.log_generator import LogGenerator + +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 + + +class GraphLogGenerator(LogGenerator): + """ + Log generator for graph. + + This is a log generator writing graph. User can use it to generate fake + summary logs about graph. + """ + + def generate_log(self, file_path, graph_dict): + """ + Generate log for external calls. + + Args: + file_path (str): Path to write logs. + graph_dict (dict): A dict consists of graph node information. + + Returns: + dict, generated scalar metadata. + + """ + + graph_event = self.generate_event(dict(graph=graph_dict)) + + self._write_log_from_event(file_path, graph_event) + + return graph_dict + + def generate_event(self, values): + """ + Method for generating graph event. + + Args: + values (dict): Graph values. e.g. {'graph': graph_dict}. + + Returns: + summary_pb2.Event. + + """ + graph_json = { + 'wall_time': time.time(), + 'graph_def': values.get('graph'), + } + graph_event = json_format.Parse(json.dumps(graph_json), summary_pb2.Event()) + + return graph_event + + +if __name__ == "__main__": + graph_log_generator = GraphLogGenerator() + test_file_name = '%s.%s.%s' % ('graph', 'summary', str(time.time())) + graph_base_path = os.path.join(os.path.dirname(__file__), os.pardir, "log_generators", "graph_base.json") + with open(graph_base_path, 'r') as load_f: + graph = json.load(load_f) + graph_log_generator.generate_log(test_file_name, graph) diff --git a/tests/ut/datavisual/utils/log_generators/images_log_generator.py b/tests/ut/datavisual/utils/log_generators/images_log_generator.py new file mode 100644 index 00000000..cf21ae1c --- /dev/null +++ b/tests/ut/datavisual/utils/log_generators/images_log_generator.py @@ -0,0 +1,166 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log generator for images.""" +import io +import time + +import numpy as np +from PIL import Image + +from tests.ut.datavisual.utils.log_generators.log_generator import LogGenerator + +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 + + +class ImagesLogGenerator(LogGenerator): + """ + Log generator for images. + + This is a log generator writing images. User can use it to generate fake + summary logs about images. + """ + + def generate_event(self, values): + """ + Method for generating image event. + + Args: + values (dict): A dict contains: + { + wall_time (float): Timestamp. + step (int): Train step. + image (np.array): Pixels tensor. + tag (str): Tag name. + } + + Returns: + summary_pb2.Event. + + """ + + image_event = summary_pb2.Event() + image_event.wall_time = values.get('wall_time') + image_event.step = values.get('step') + + height, width, channel, image_string = self._get_image_string(values.get('image')) + value = image_event.summary.value.add() + value.tag = values.get('tag') + value.image.height = height + value.image.width = width + value.image.colorspace = channel + value.image.encoded_image = image_string + + return image_event + + def _get_image_string(self, image_tensor): + """ + Generate image string from tensor. + + Args: + image_tensor (np.array): Pixels tensor. + + Returns: + int, height. + int, width. + int, channel. + bytes, image_string. + + """ + height, width, channel = image_tensor.shape + scaled_height = int(height) + scaled_width = int(width) + + image = Image.fromarray(image_tensor) + image = image.resize((scaled_width, scaled_height), Image.ANTIALIAS) + output = io.BytesIO() + image.save(output, format='PNG') + image_string = output.getvalue() + output.close() + return height, width, channel, image_string + + def _make_image_tensor(self, shape): + """ + Make image tensor according to shape. + + Args: + shape (list): Shape of image, consists of height, width, channel. + + Returns: + np.array, image tensor. + + """ + image = np.prod(shape) + image_tensor = (np.arange(image, dtype=float)).reshape(shape) + image_tensor = image_tensor / np.max(image_tensor) * 255 + image_tensor = image_tensor.astype(np.uint8) + + return image_tensor + + def generate_log(self, file_path, steps_list, tag_name): + """ + Generate log for external calls. + + Args: + file_path (str): Path to write logs. + steps_list (list): A list consists of step. + tag_name (str): Tag name. + + Returns: + list[dict], generated image metadata. + dict, generated image tensors. + + """ + + images_values = dict() + images_metadata = [] + for step in steps_list: + wall_time = time.time() + + # height, width, channel + image_tensor = self._make_image_tensor([5, 5, 3]) + + image_metadata = dict() + image_metadata.update({'wall_time': wall_time}) + image_metadata.update({'step': step}) + image_metadata.update({'height': image_tensor.shape[0]}) + image_metadata.update({'width': image_tensor.shape[1]}) + images_metadata.append(image_metadata) + images_values.update({step: image_tensor}) + + values = dict( + wall_time=wall_time, + step=step, + image=image_tensor, + tag=tag_name + ) + + self._write_log_one_step(file_path, values) + + return images_metadata, images_values + + def get_image_tensor_from_bytes(self, image_string): + """Get image tensor from bytes.""" + img = Image.open(io.BytesIO(image_string)) + image_tensor = np.array(img) + + return image_tensor + + +if __name__ == "__main__": + images_log_generator = ImagesLogGenerator() + test_file_name = '%s.%s.%s' % ('image', 'summary', str(time.time())) + test_steps = [1, 3, 5] + test_tags = "test_image_tag_name" + images_log_generator.generate_log(test_file_name, test_steps, test_tags) diff --git a/tests/ut/datavisual/utils/log_generators/log_generator.py b/tests/ut/datavisual/utils/log_generators/log_generator.py new file mode 100644 index 00000000..d9b1653f --- /dev/null +++ b/tests/ut/datavisual/utils/log_generators/log_generator.py @@ -0,0 +1,75 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base log Generator.""" + +import struct +from abc import abstractmethod + +from tests.ut.datavisual.utils import crc32 + + +class LogGenerator: + """ + Base log generator. + + This is a base class for log generators. User can use it to generate fake + summary logs. + """ + + @abstractmethod + def generate_event(self, values): + """ + Abstract method for generating event. + + Args: + values (dict): Values. + + Returns: + summary_pb2.Event. + + """ + + def _write_log_one_step(self, file_path, values): + """ + Write log one step. + + Args: + file_path (str): File path to write. + values (dict): Values. + + """ + event = self.generate_event(values) + self._write_log_from_event(file_path, event) + + @staticmethod + def _write_log_from_event(file_path, event): + """ + Write log by event. + + Args: + file_path (str): File path to write. + event (summary_pb2.Event): Event object in proto. + + """ + send_msg = event.SerializeToString() + + header = struct.pack(' time_limit: + break + time.sleep(0.1) + continue diff --git a/tests/ut/lineagemgr/__init__.py b/tests/ut/lineagemgr/__init__.py new file mode 100644 index 00000000..75c8ea32 --- /dev/null +++ b/tests/ut/lineagemgr/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Import the mocked mindspore.""" +import sys +from .collection.model import mindspore + +sys.modules['mindspore'] = mindspore diff --git a/tests/ut/lineagemgr/api/__init__.py b/tests/ut/lineagemgr/api/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/api/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/api/test_model.py b/tests/ut/lineagemgr/api/test_model.py new file mode 100644 index 00000000..6dccde6a --- /dev/null +++ b/tests/ut/lineagemgr/api/test_model.py @@ -0,0 +1,253 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the model module.""" +from unittest import TestCase, mock +from unittest.mock import MagicMock + +from mindinsight.lineagemgr import get_summary_lineage, filter_summary_lineage +from mindinsight.lineagemgr.api.model import _convert_relative_path_to_abspath +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamSummaryPathError, \ + LineageFileNotFoundError, LineageSummaryParseException, LineageQuerierParamException, \ + LineageQuerySummaryDataError, LineageSearchConditionParamError, LineageParamTypeError, \ + LineageParamValueError +from mindinsight.lineagemgr.common.path_parser import SummaryPathParser + + +class TestModel(TestCase): + """Test the function of get_summary_lineage and filter_summary_lineage.""" + + @mock.patch('mindinsight.lineagemgr.api.model.Querier') + @mock.patch('mindinsight.lineagemgr.api.model.SummaryPathParser.get_latest_lineage_summary') + @mock.patch('os.path.isdir') + def test_get_summary_lineage_success(self, isdir_mock, latest_summary_mock, qurier_mock): + """Test the function of get_summary_lineage.""" + isdir_mock.return_value = True + latest_summary_mock.return_value = '/path/to/summary_dir/a_MS_lineage' + + mock_querier = MagicMock() + qurier_mock.return_value = mock_querier + mock_querier.get_summary_lineage.return_value = [{'algorithm': {'network': 'ResNet'}}] + summary_dir = '/path/to/summary_dir' + result = get_summary_lineage(summary_dir, keys=['algorithm']) + self.assertEqual(result, {'algorithm': {'network': 'ResNet'}}) + + def test_get_summary_lineage_failed(self): + """Test get_summary_lineage failed.""" + invalid_path = '../fake_dir' + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + get_summary_lineage, + invalid_path + ) + + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + @mock.patch.object(SummaryPathParser, 'get_latest_lineage_summary') + def test_get_summary_lineage_failed2(self, mock_summary, mock_valid): + """Test get_summary_lineage failed.""" + mock_summary.return_value = None + mock_valid.return_value = '/path/to/summary/dir' + self.assertRaisesRegex( + LineageFileNotFoundError, + 'no summary log file under summary_dir', + get_summary_lineage, + '/path/to/summary_dir' + ) + + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + @mock.patch('mindinsight.lineagemgr.api.model.Querier') + @mock.patch.object(SummaryPathParser, 'get_latest_lineage_summary') + def test_get_summary_lineage_failed3(self, + mock_summary, + mock_querier, + mock_valid): + """Test get_summary_lineage failed.""" + mock_summary.return_value = '/path/to/summary/file' + mock_querier.return_value.get_summary_lineage.side_effect = \ + LineageSummaryParseException() + mock_valid.return_value = '/path/to/summary_dir' + res = get_summary_lineage('/path/to/summary_dir') + assert res == {} + + mock_querier.side_effect = LineageQuerierParamException( + ['keys'], 'key') + self.assertRaisesRegex( + LineageQuerySummaryDataError, + 'Get summary lineage failed', + get_summary_lineage, + '/path/to/summary_dir' + ) + + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + def test_convert_relative_path_to_abspath(self, validate_path_mock): + """Test the function of converting realtive path to abspath.""" + validate_path_mock.return_value = '/path/to/summary_base_dir/summary_dir' + summary_base_dir = '/path/to/summary_base_dir' + search_condition = { + 'summary_dir': { + 'in': ['/path/to/summary_base_dir'] + } + } + result = _convert_relative_path_to_abspath(summary_base_dir, + search_condition) + self.assertDictEqual( + result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}}) + + search_condition = { + 'summary_dir': { + 'in': ['./summary_dir'] + } + } + result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) + self.assertDictEqual( + result, {'summary_dir': {'in': ['/path/to/summary_base_dir/summary_dir']}} + ) + + search_condition = { + 'summary_dir': { + 'eq': '/summary_dir' + } + } + result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) + self.assertDictEqual( + result, {'summary_dir': {'eq': '/path/to/summary_base_dir/summary_dir'}}) + + search_condition = { + 'summary_dir': None + } + result = _convert_relative_path_to_abspath(summary_base_dir, search_condition) + self.assertDictEqual( + result, search_condition + ) + search_condition = { + 'summary_dir': { + 'gt': 3 + } + } + self.assertRaisesRegex( + LineageParamValueError, + 'Invalid operation of summary dir', + _convert_relative_path_to_abspath, + summary_base_dir, + search_condition + ) + + +class TestFilterAPI(TestCase): + """Test the function of filter_summary_lineage.""" + + @mock.patch('mindinsight.lineagemgr.api.model.Querier') + @mock.patch('mindinsight.lineagemgr.api.model.SummaryPathParser.get_latest_lineage_summaries') + @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath') + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock, + latest_summary_mock, qurier_mock): + """Test the function of filter_summary_lineage.""" + convert_path_mock.return_value = { + 'summary_dir': { + 'in': ['/path/to/summary_base_dir'] + }, + 'loss': { + 'gt': 2.0 + } + } + validate_path_mock.return_value = True + latest_summary_mock.return_value = ['/path/to/summary_base_dir/summary_dir'] + mock_querier = MagicMock() + qurier_mock.return_value = mock_querier + mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}] + + summary_base_dir = '/path/to/summary_base_dir' + result = filter_summary_lineage(summary_base_dir) + self.assertEqual(result, [{'loss': 3.0}]) + + def test_invalid_path(self): + """Test filter_summary_lineage with invalid path.""" + invalid_path = '../fake_dir' + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'The summary path is invalid.', + filter_summary_lineage, + invalid_path + ) + + @mock.patch('mindinsight.lineagemgr.api.model.validate_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + def test_invalid_search_condition(self, mock_path, mock_valid): + """Test filter_summary_lineage with invalid invalid param.""" + mock_path.return_value = None + mock_valid.side_effect = LineageParamTypeError( + 'Invalid search_condition type.') + self.assertRaisesRegex( + LineageSearchConditionParamError, + 'Invalid search_condition type.', + filter_summary_lineage, + '/path/to/summary/dir', + 'invalid_condition' + ) + + @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath') + def test_failed_to_convert_path(self, mock_convert, *args): + """Test filter_summary_lineage with invalid invalid param.""" + mock_convert.side_effect = LineageParamValueError('invalid path') + args[0].return_value = None + self.assertRaisesRegex( + LineageParamSummaryPathError, + 'invalid path', + filter_summary_lineage, + '/path/to/summary/dir', + {} + ) + + @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath') + @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + @mock.patch.object(SummaryPathParser, 'get_latest_lineage_summaries') + def test_failed_to_get_summary_filesh(self, mock_parse, *args): + """Test filter_summary_lineage with invalid invalid param.""" + mock_parse.return_value = [] + args[0].return_value = None + self.assertRaisesRegex( + LineageFileNotFoundError, + 'There is no summary log file under summary_base_dir.', + filter_summary_lineage, + '/path/to/summary/dir' + ) + + @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath') + @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_condition') + @mock.patch('mindinsight.lineagemgr.api.model.validate_path') + @mock.patch.object(SummaryPathParser, 'get_latest_lineage_summaries') + @mock.patch('mindinsight.lineagemgr.api.model.Querier') + def test_failed_to_querier(self, mock_query, mock_parse, *args): + """Test filter_summary_lineage with invalid invalid param.""" + mock_query.side_effect = LineageSummaryParseException() + mock_parse.return_value = ['/path/to/summary/file'] + args[0].return_value = None + res = filter_summary_lineage('/path/to/summary') + assert res == {'object': [], 'count': 0} + + mock_query.side_effect = LineageQuerierParamException(['keys'], 'key') + self.assertRaisesRegex( + LineageQuerySummaryDataError, + 'Filter summary lineage failed.', + filter_summary_lineage, + '/path/to/summary/dir' + ) diff --git a/tests/ut/lineagemgr/collection/__init__.py b/tests/ut/lineagemgr/collection/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/__init__.py b/tests/ut/lineagemgr/collection/model/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/__init__.py new file mode 100644 index 00000000..eea7413a --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore Interface.""" +from .application.model_zoo.resnet import ResNet +from .common.tensor import Tensor +from .dataset import MindDataset +from .nn import * +from .train.callback import _ListCallback, Callback, RunContext, ModelCheckpoint, SummaryStep +from .train.summary import SummaryRecord diff --git a/tests/ut/lineagemgr/collection/model/mindspore/application/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/application/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/application/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py b/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py new file mode 100644 index 00000000..f9755ec9 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/application/model_zoo/resnet.py @@ -0,0 +1,24 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore ResNet class.""" +from ...nn.cell import Cell + + +class ResNet(Cell): + """Mocked ResNet.""" + + def __init__(self): + super(ResNet, self).__init__() + self._cells = {} diff --git a/tests/ut/lineagemgr/collection/model/mindspore/common/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/common/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/common/tensor.py b/tests/ut/lineagemgr/collection/model/mindspore/common/tensor.py new file mode 100644 index 00000000..33205845 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/common/tensor.py @@ -0,0 +1,30 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/common/tensor.py.""" +import numpy as np + + +class Tensor: + """Mock the MindSpore Tensor class.""" + + def __init__(self, value=0): + self._value = value + + def asnumpy(self): + """Get value in numpy format.""" + return np.array(self._value) + + def __repr__(self): + return str(self.asnumpy()) diff --git a/tests/ut/lineagemgr/collection/model/mindspore/context.py b/tests/ut/lineagemgr/collection/model/mindspore/context.py new file mode 100644 index 00000000..e3dbef0e --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/context.py @@ -0,0 +1,21 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" + + +def get_context(key): + """Get key in context.""" + context = {"device_id": 1} + return context.get(key) diff --git a/tests/ut/lineagemgr/collection/model/mindspore/dataset/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/dataset/__init__.py new file mode 100644 index 00000000..e8f33976 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/dataset/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock mindspore.dataset.""" +from .engine import MindDataset diff --git a/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py new file mode 100644 index 00000000..562542fd --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock mindspore.dataset.engine.""" +from .datasets import MindDataset, Dataset diff --git a/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py b/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py new file mode 100644 index 00000000..a9decbce --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/dataset/engine/datasets.py @@ -0,0 +1,36 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/dataset/engine/datasets.py.""" + + +class Dataset: + """Mock the MindSpore Dataset class.""" + + def __init__(self, dataset_size=None, dataset_path=None): + self.dataset_size = dataset_size + self.dataset_path = dataset_path + self.input = [] + + def get_dataset_size(self): + """Mocked get_dataset_size.""" + return self.dataset_size + + +class MindDataset(Dataset): + """Mock the MindSpore MindDataset class.""" + + def __init__(self, dataset_size=None, dataset_file=None): + super(MindDataset, self).__init__(dataset_size) + self.dataset_file = dataset_file diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/__init__.py new file mode 100644 index 00000000..4b8053a2 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the mindspore.nn package.""" +from .optim import Optimizer, Momentum +from .loss.loss import SoftmaxCrossEntropyWithLogits, _Loss +from .cell import Cell, WithLossCell, TrainOneStepWithLossScaleCell + +__all__ = ['Optimizer', 'Momentum', 'SoftmaxCrossEntropyWithLogits', + '_Loss', 'Cell', 'WithLossCell', + 'TrainOneStepWithLossScaleCell'] diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/cell.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/cell.py new file mode 100644 index 00000000..f1e19c3f --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/cell.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/train/callback.py.""" + + +class Cell: + """Mock the Cell class.""" + + def __init__(self, auto_prefix=True, pips=None): + if pips is None: + pips = dict() + self._auto_prefix = auto_prefix + self._pips = pips + + @property + def auto_prefix(self): + """The property of auto_prefix.""" + return self._auto_prefix + + @property + def pips(self): + """The property of pips.""" + return self._pips + + +class WithLossCell(Cell): + """Mocked WithLossCell class.""" + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__(auto_prefix=False, pips=backbone.pips) + self._backbone = backbone + self._loss_fn = loss_fn + + +class TrainOneStepWithLossScaleCell(Cell): + """Mocked TrainOneStepWithLossScaleCell.""" + + def __init__(self): + super(TrainOneStepWithLossScaleCell, self).__init__() diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/loss.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/loss.py new file mode 100644 index 00000000..7323a087 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/loss/loss.py @@ -0,0 +1,39 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore SoftmaxCrossEntropyWithLogits class.""" +from ..cell import Cell + + +class _Loss(Cell): + """Mocked _Loss.""" + + def __init__(self, reduction='mean'): + super(_Loss, self).__init__() + self.reduction = reduction + + def construct(self, base, target): + """Mocked construct function.""" + raise NotImplementedError + + +class SoftmaxCrossEntropyWithLogits(_Loss): + """Mocked SoftmaxCrossEntropyWithLogits.""" + + def __init__(self, weight=None): + super(SoftmaxCrossEntropyWithLogits, self).__init__(weight) + + def construct(self, base, target): + """Mocked construct.""" + return 1 diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/optim.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/optim.py new file mode 100644 index 00000000..84e9a8ab --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/optim.py @@ -0,0 +1,49 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/nn/optim.py.""" +from .cell import Cell + + +class Parameter: + """Mock the MindSpore Parameter class.""" + + def __init__(self, learning_rate): + self._name = "Parameter" + self.default_input = learning_rate + + @property + def name(self): + """The property of name.""" + return self._name + + def __repr__(self): + format_str = 'Parameter (name={name})' + return format_str.format(name=self._name) + + +class Optimizer(Cell): + """Mock the MindSpore Optimizer class.""" + + def __init__(self, learning_rate): + super(Optimizer, self).__init__() + self.learning_rate = Parameter(learning_rate) + + +class Momentum(Optimizer): + """Mock the MindSpore Momentum class.""" + + def __init__(self, learning_rate): + super(Momentum, self).__init__(learning_rate) + self.dynamic_lr = False diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py new file mode 100644 index 00000000..6d725bbe --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore wrap package.""" +from .loss_scale import TrainOneStepWithLossScaleCell +from .cell_wrapper import WithLossCell diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py new file mode 100644 index 00000000..49b84e42 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/cell_wrapper.py @@ -0,0 +1,25 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore cell_wrapper.py.""" +from ..cell import Cell + + +class WithLossCell(Cell): + """Mock the WithLossCell class.""" + + def __init__(self, backbone, loss_fn): + super(WithLossCell, self).__init__() + self._backbone = backbone + self._loss_fn = loss_fn diff --git a/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py new file mode 100644 index 00000000..0473aacb --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/nn/wrap/loss_scale.py @@ -0,0 +1,29 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock MindSpore loss_scale.py.""" +from ..cell import Cell + + +class TrainOneStepWithLossScaleCell(Cell): + """Mock the TrainOneStepWithLossScaleCell class.""" + + def __init__(self, network, optimizer): + super(TrainOneStepWithLossScaleCell, self).__init__() + self.network = network + self.optimizer = optimizer + + def construct(self, data, label): + """Mock the construct method.""" + raise NotImplementedError diff --git a/tests/ut/lineagemgr/collection/model/mindspore/train/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/train/__init__.py new file mode 100644 index 00000000..0809f353 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/train/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/collection/model/mindspore/train/callback.py b/tests/ut/lineagemgr/collection/model/mindspore/train/callback.py new file mode 100644 index 00000000..7b1d39f2 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/train/callback.py @@ -0,0 +1,84 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Mock the MindSpore mindspore/train/callback.py.""" +import os + + +class RunContext: + """Mock the RunContext class.""" + + def __init__(self, original_args=None): + self._original_args = original_args + self._stop_requested = False + + def original_args(self): + """Mock original_args.""" + return self._original_args + + def stop_requested(self): + """Mock stop_requested method.""" + return self._stop_requested + + +class Callback: + """Mock the Callback class.""" + + def __init__(self): + pass + + def begin(self, run_context): + """Called once before network training.""" + + def epoch_begin(self, run_context): + """Called before each epoch begin.""" + + +class _ListCallback(Callback): + """Mock the _ListCallabck class.""" + + def __init__(self, callbacks): + super(_ListCallback, self).__init__() + self._callbacks = callbacks + + +class ModelCheckpoint(Callback): + """Mock the ModelCheckpoint class.""" + + def __init__(self, prefix='CKP', directory=None, config=None): + super(ModelCheckpoint, self).__init__() + self._prefix = prefix + self._directory = directory + self._config = config + self._latest_ckpt_file_name = os.path.join(directory, prefix + 'test_model.ckpt') + + @property + def model_file_name(self): + """Get the file name of model.""" + return self._model_file_name + + @property + def latest_ckpt_file_name(self): + """Get the latest file name fo checkpoint.""" + return self._latest_ckpt_file_name + + +class SummaryStep(Callback): + """Mock the SummaryStep class.""" + + def __init__(self, summary, flush_step=10): + super(SummaryStep, self).__init__() + self._sumamry = summary + self._flush_step = flush_step + self.summary_file_name = summary.full_file_name diff --git a/tests/ut/lineagemgr/collection/model/mindspore/train/summary/__init__.py b/tests/ut/lineagemgr/collection/model/mindspore/train/summary/__init__.py new file mode 100644 index 00000000..ed0eee75 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/train/summary/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" +from .summary_record import SummaryRecord + +__all__ = ["SummaryRecord"] diff --git a/tests/ut/lineagemgr/collection/model/mindspore/train/summary/summary_record.py b/tests/ut/lineagemgr/collection/model/mindspore/train/summary/summary_record.py new file mode 100644 index 00000000..1e43bffb --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/mindspore/train/summary/summary_record.py @@ -0,0 +1,38 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MindSpore Mock Interface""" +import os +import time + + +class SummaryRecord: + """Mock the MindSpore SummaryRecord class.""" + + def __init__(self, + log_dir: str, + file_prefix: str = "events.", + file_suffix: str = ".MS", + create_time=int(time.time())): + self.log_dir = log_dir + self.prefix = file_prefix + self.suffix = file_suffix + file_name = file_prefix + 'summary.' + str(create_time) + file_suffix + self.full_file_name = os.path.join(log_dir, file_name) + + def flush(self): + """Mock flush method.""" + + def close(self): + """Mock close method.""" diff --git a/tests/ut/lineagemgr/collection/model/test_model_lineage.py b/tests/ut/lineagemgr/collection/model/test_model_lineage.py new file mode 100644 index 00000000..faab5634 --- /dev/null +++ b/tests/ut/lineagemgr/collection/model/test_model_lineage.py @@ -0,0 +1,463 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Unittest for model_lineage.py""" +import os +import shutil +import unittest +from unittest import mock, TestCase +from unittest.mock import MagicMock + +from mindinsight.lineagemgr.collection.model.model_lineage import TrainLineage, EvalLineage, \ + AnalyzeObject +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageLogError, LineageGetModelFileError, MindInsightException +from mindspore.common.tensor import Tensor +from mindspore.dataset.engine import MindDataset, Dataset +from mindspore.nn import Optimizer, WithLossCell, TrainOneStepWithLossScaleCell, \ + SoftmaxCrossEntropyWithLogits +from mindspore.train.callback import RunContext, ModelCheckpoint, SummaryStep +from mindspore.train.summary import SummaryRecord + + +class TestModelLineage(TestCase): + """Test TrainLineage and EvalLineage class in model_lineage.py.""" + + @classmethod + def setUpClass(cls): + cls.lineage_list = ['train_network', 'loss_fn', 'optimizer', 'train_dataset', + 'valid_dataset', 'epoch', 'valid_step', + 'hybrid_parallel', 'data_parallel_size', 'auto_parallel', + 'device_number', 'batch_num', 'summary_log_path', + 'model_ckpt'] + cls.run_context = {key: None for key in cls.lineage_list} + cls.run_context['net_outputs'] = Tensor() + cls.my_run_context = RunContext + cls.my_train_module = TrainLineage + cls.my_eval_module = EvalLineage + cls.my_analyze_module = AnalyzeObject + cls.my_summary_record = SummaryRecord + cls.summary_log_path = '/path/to/summary_log' + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_summary_record_exception(self, mock_validate_summary): + """Test SummaryRecord with exception.""" + mock_validate_summary.return_value = None + summary_record = self.my_summary_record(self.summary_log_path) + with self.assertRaises(MindInsightException) as context: + self.my_train_module(summary_record=summary_record, raise_exception=1) + self.assertTrue(f'Invalid value for raise_exception.' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.ds') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'LineageSummary.record_dataset_graph') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'validate_summary_record') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.get_optimizer_by_network') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.analyze_optimizer') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network') + def test_begin(self, *args): + """Test TrainLineage.begin method.""" + args[1].return_value = None + args[2].return_value = Optimizer(Tensor(0.1)) + args[3].return_value = None + args[5].serialize.return_value = {} + run_context = {'optimizer': Optimizer(Tensor(0.1)), + 'epoch_num': 10} + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path)) + train_lineage.begin(self.my_run_context(run_context)) + args[4].assert_called() + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.ds') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'LineageSummary.record_dataset_graph') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'validate_summary_record') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.get_optimizer_by_network') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.analyze_optimizer') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network') + def test_begin_error(self, *args): + """Test TrainLineage.begin method.""" + args[1].return_value = None + args[2].return_value = Optimizer(Tensor(0.1)) + args[3].return_value = None + args[4].side_effect = Exception + args[5].serialize.return_value = {} + run_context = {'optimizer': Optimizer(Tensor(0.1)), + 'epoch_num': 10} + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaisesRegex(LineageLogError, 'Dataset graph log error'): + train_lineage.begin(self.my_run_context(run_context)) + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path)) + train_lineage.begin(self.my_run_context(run_context)) + args[4].assert_called() + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_begin_exception(self, *args): + """Test TrainLineage.begin method with exception.""" + args[0].return_value = None + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(Exception) as context: + train_lineage.begin(self.run_context) + self.assertTrue('Invalid TrainLineage run_context.' in str(context.exception)) + + run_context = {key: None for key in self.lineage_list} + run_context['optimizer'] = 1 + with self.assertRaises(Exception) as context: + train_lineage.begin(self.my_run_context(run_context)) + self.assertTrue('The parameter optimizer is invalid.' in str(context.exception)) + + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context') + @mock.patch('builtins.float') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_train_end(self, *args): + """Test TrainLineage.end method.""" + args[1].return_value = 2.0 + args[2].return_value = True + args[3].return_value = True + args[4].return_value = None + args[5].return_value = None + args[6].return_value = None + args[7].return_value = (None, None) + args[8].return_value = 10 + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + train_lineage.end(self.my_run_context(self.run_context)) + args[6].assert_called() + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_train_end_exception(self, mock_validate_summary): + """Test TrainLineage.end method when exception.""" + mock_validate_summary.return_value = True + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(Exception) as context: + train_lineage.end(self.run_context) + self.assertTrue('Invalid TrainLineage run_context.' in str(context.exception)) + + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context') + @mock.patch('builtins.float') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_train_end_exception_log_error(self, *args): + """Test TrainLineage.end method with logging errors.""" + args[1].return_value = 2.0 + args[2].return_value = True + args[3].return_value = True + args[4].return_value = None + args[5].return_value = None + args[6].side_effect = Exception + args[7].return_value = (None, None) + args[8].return_value = 10 + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(LineageLogError) as context: + train_lineage.end(self.my_run_context(self.run_context)) + self.assertTrue('End error in TrainLineage:' in str(context.exception)) + + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_model_size') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_path') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.LineageSummary.record_train_lineage') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_dataset') + @mock.patch( + 'mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.analyze_optimizer') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_network') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_train_run_context') + @mock.patch('builtins.float') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_train_end_exception_log_error2(self, *args): + """Test TrainLineage.end method with logging errors.""" + args[1].return_value = 2.0 + args[2].return_value = True + args[3].return_value = True + args[4].return_value = None + args[5].return_value = None + args[6].side_effect = IOError + args[7].return_value = (None, None) + args[8].return_value = 10 + run_context = {key: None for key in self.lineage_list} + run_context['loss_fn'] = MagicMock() + run_context['net_outputs'] = Tensor(0.11) + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(LineageLogError) as context: + train_lineage.end(self.my_run_context(run_context)) + self.assertTrue('End error in TrainLineage:' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_eval_exception_train_id_none(self, mock_validate_summary): + """Test EvalLineage.end method with initialization error.""" + mock_validate_summary.return_value = True + with self.assertRaises(MindInsightException) as context: + self.my_eval_module(self.my_summary_record(self.summary_log_path), raise_exception=2) + self.assertTrue('Invalid value for raise_exception.' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.analyze_dataset') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_eval_run_context') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'LineageSummary.record_evaluation_lineage') + def test_eval_end(self, *args): + """Test EvalLineage.end method.""" + args[1].return_value = True + args[2].return_value = True + args[3].return_value = None + args[0].return_value = None + eval_lineage = self.my_eval_module(self.my_summary_record(self.summary_log_path)) + eval_lineage.end(self.my_run_context(self.run_context)) + args[0].assert_called() + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + def test_eval_end_except_run_context(self, mock_validate_summary): + """Test EvalLineage.end method when run_context is invalid..""" + mock_validate_summary.return_value = True + eval_lineage = self.my_eval_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(Exception) as context: + eval_lineage.end(self.run_context) + self.assertTrue('Invalid EvalLineage run_context.' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.analyze_dataset') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_eval_run_context') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'LineageSummary.record_evaluation_lineage') + def test_eval_end_except_log_error(self, *args): + """Test EvalLineage.end method with logging error.""" + args[0].side_effect = Exception + args[1].return_value = True + args[2].return_value = True + args[3].return_value = None + eval_lineage = self.my_eval_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(LineageLogError) as context: + eval_lineage.end(self.my_run_context(self.run_context)) + self.assertTrue('End error in EvalLineage' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.analyze_dataset') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_summary_record') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.validate_eval_run_context') + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'LineageSummary.record_evaluation_lineage') + def test_eval_end_except_log_error2(self, *args): + """Test EvalLineage.end method with logging error.""" + args[0].side_effect = IOError + args[1].return_value = True + args[2].return_value = True + args[3].return_value = None + eval_lineage = self.my_eval_module(self.my_summary_record(self.summary_log_path), True) + with self.assertRaises(LineageLogError) as context: + eval_lineage.end(self.my_run_context(self.run_context)) + self.assertTrue('End error in EvalLineage' in str(context.exception)) + + def test_epoch_is_zero(self): + """Test TrainLineage.end method.""" + run_context = self.run_context + run_context['epoch_num'] = 0 + with self.assertRaises(MindInsightException): + train_lineage = self.my_train_module(self.my_summary_record(self.summary_log_path), True) + train_lineage.end(self.my_run_context(run_context)) + + def tearDown(self): + """Teardown.""" + if os.path.exists(self.summary_log_path): + try: + shutil.rmtree(self.summary_log_path) + except IOError: + pass + + +class TestAnalyzer(TestCase): + """Test Analyzer class in model_lineage.py.""" + + def setUp(self): + """SetUp config.""" + self.analyzer = AnalyzeObject() + + def test_analyze_optimizer(self): + """Test analyze_optimizer method.""" + optimizer = Optimizer(Tensor(0.12)) + res = self.analyzer.analyze_optimizer(optimizer) + assert res == 0.12 + + def test_get_dataset_path(self): + """Test get_dataset_path method.""" + dataset = MindDataset( + dataset_file='/path/to/mindrecord' + ) + res = self.analyzer.get_dataset_path(dataset) + assert res == '/path/to/mindrecord' + + def test_get_dataset_path_wrapped(self): + """Test get_dataset_path_wrapped method.""" + dataset = Dataset() + dataset.input.append( + MindDataset( + dataset_size=10, + dataset_file='/path/to/cifar10' + )) + + res = self.analyzer.get_dataset_path_wrapped(dataset) + assert res == '/path/to/cifar10' + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.' + 'AnalyzeObject.get_dataset_path_wrapped') + def test_analyze_dataset(self, mock_get_path): + """Test analyze_dataset method.""" + mock_get_path.return_value = '/path/to/mindinsightset' + dataset = MindDataset( + dataset_size=10, + dataset_file='/path/to/mindinsightset' + ) + res1 = self.analyzer.analyze_dataset(dataset, {'step_num': 10, 'epoch': 2}, 'train') + res2 = self.analyzer.analyze_dataset(dataset, {'step_num': 5}, 'valid') + assert res1 == {'step_num': 10, + 'train_dataset_path': '/path/to', + 'train_dataset_size': 50, + 'epoch': 2} + assert res2 == {'step_num': 5, 'valid_dataset_path': '/path/to', + 'valid_dataset_size': 50} + + def test_get_dataset_path_dataset(self): + """Test get_dataset_path method with Dataset.""" + dataset = Dataset( + dataset_size=10, + dataset_path='/path/to/cifar10' + ) + + with self.assertRaises(IndexError): + self.analyzer.get_dataset_path(output_dataset=dataset) + + def test_get_dataset_path_mindrecord(self): + """Test get_dataset_path method with MindDataset.""" + dataset = MindDataset( + dataset_file='/path/to/cifar10' + ) + dataset_path = self.analyzer.get_dataset_path(output_dataset=dataset) + self.assertEqual(dataset_path, '/path/to/cifar10') + + def test_get_file_path(self): + """Test get_file_path method.""" + model_ckpt = ModelCheckpoint(prefix='', directory='/path/to') + summary_step = SummaryStep(MagicMock(full_file_name='/path/to/summary.log')) + list_callback = [model_ckpt, summary_step] + ckpt_file_path, _ = AnalyzeObject.get_file_path(list_callback) + self.assertEqual(ckpt_file_path, '/path/to/test_model.ckpt') + + @mock.patch('os.path.getsize') + def test_get_file_size(self, os_get_size_mock): + """Test get_file_size method.""" + os_get_size_mock.return_value = 128 + file_size = AnalyzeObject.get_file_size('/file/path') + self.assertEqual(file_size, 128) + + @mock.patch('os.path.getsize') + def test_get_file_size_except(self, os_get_size_mock): + """Test failed to get the size of file.""" + os_get_size_mock.side_effect = OSError + analyzer = AnalyzeObject + with self.assertRaises(LineageGetModelFileError) as context: + analyzer.get_file_size('/file/path') + self.assertTrue('Error when get model file size:' in str(context.exception)) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_size') + def test_get_model_size(self, get_file_size_mock): + """Test get_model_size method.""" + get_file_size_mock.return_value = 128 + analyzer = AnalyzeObject + file_size = analyzer.get_model_size(ckpt_file_path='/file/path') + self.assertEqual(file_size, 128) + + @mock.patch('mindinsight.lineagemgr.collection.model.model_lineage.AnalyzeObject.get_file_size') + def test_get_model_size_no_ckpt(self, get_file_size_mock): + """Test get_model_size method without ckpt file.""" + get_file_size_mock.return_value = 0 + analyzer = AnalyzeObject + file_size = analyzer.get_model_size(ckpt_file_path='') + self.assertEqual(file_size, 0) + + @mock.patch('builtins.vars') + def test_get_optimizer_by_network(self, mock_vars): + """Test get_optimizer_by_network.""" + mock_optimizer = Optimizer(Tensor(0.1)) + mock_cells = MagicMock() + mock_cells.items.return_value = [{'key': mock_optimizer}] + mock_vars.return_value = { + '_cells': { + 'key': mock_optimizer + } + } + res = AnalyzeObject.get_optimizer_by_network(MagicMock()) + self.assertEqual(res, mock_optimizer) + + @mock.patch('builtins.vars') + def test_get_loss_fn_by_network(self, mock_vars): + """Test get_loss_fn_by_network.""" + mock_cell1 = {'_cells': {'key': SoftmaxCrossEntropyWithLogits(0.2)}} + mock_cell2 = {'_cells': {'opt': Optimizer(Tensor(0.1))}} + mock_cell3 = {'_cells': {'loss': SoftmaxCrossEntropyWithLogits(0.1)}} + mock_vars.side_effect = [mock_cell1, mock_cell2, mock_cell3] + res = AnalyzeObject.get_loss_fn_by_network(MagicMock()) + self.assertEqual(res, mock_cell3['_cells']['loss']) + + @mock.patch('builtins.vars') + def test_get_backbone_network_with_loss_cell(self, mock_vars): + """Test get_backbone_network with loss_cell.""" + mock_cell = {'_cells': {'key': WithLossCell(MagicMock(), + SoftmaxCrossEntropyWithLogits(0.1))} + } + mock_vars.return_value = mock_cell + res = AnalyzeObject.get_backbone_network(MagicMock()) + self.assertEqual(res, 'MagicMock') + + @mock.patch('builtins.vars') + def test_get_backbone_network(self, mock_vars): + """Test get_backbone_network.""" + mock_net = TrainOneStepWithLossScaleCell() + mock_net.network = MagicMock() + mock_cell = { + '_cells': { + 'key': mock_net + } + } + mock_vars.return_value = mock_cell + res = AnalyzeObject.get_backbone_network(MagicMock()) + self.assertEqual(res, 'MagicMock') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/tests/ut/lineagemgr/common/__init__.py b/tests/ut/lineagemgr/common/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/lineagemgr/common/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/common/test_path_parser.py b/tests/ut/lineagemgr/common/test_path_parser.py new file mode 100644 index 00000000..817a667b --- /dev/null +++ b/tests/ut/lineagemgr/common/test_path_parser.py @@ -0,0 +1,168 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the path_parser module.""" +from datetime import datetime +from unittest import TestCase, mock + +from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher +from mindinsight.lineagemgr.common.path_parser import SummaryPathParser + + +class TestSummaryPathParser(TestCase): + """Test the class of SummaryPathParser.""" + + @mock.patch.object(SummaryWatcher, 'list_summary_directories') + def test_get_summary_dirs(self, *args): + """Test the function of get_summary_dirs.""" + args[0].return_value = [ + { + 'relative_path': './relative_path0' + }, + { + 'relative_path': './' + }, + { + 'relative_path': './relative_path1' + } + ] + + expected_result = [ + '/path/to/base/relative_path0', + '/path/to/base', + '/path/to/base/relative_path1' + ] + base_dir = '/path/to/base' + result = SummaryPathParser.get_summary_dirs(base_dir) + self.assertListEqual(expected_result, result) + + args[0].return_value = [] + result = SummaryPathParser.get_summary_dirs(base_dir) + self.assertListEqual([], result) + + @mock.patch.object(SummaryWatcher, 'list_summaries') + def test_get_latest_lineage_summary(self, *args): + """Test the function of get_latest_lineage_summary.""" + args[0].return_value = [ + { + 'file_name': 'file0', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file0_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file1', + 'create_time': datetime.fromtimestamp(1582031971) + }, + { + 'file_name': 'file1_lineage', + 'create_time': datetime.fromtimestamp(1582031971) + } + ] + summary_dir = '/path/to/summary_dir' + result = SummaryPathParser.get_latest_lineage_summary(summary_dir) + self.assertEqual('/path/to/summary_dir/file1_lineage', result) + + args[0].return_value = [ + { + 'file_name': 'file0', + 'create_time': datetime.fromtimestamp(1582031970) + } + ] + result = SummaryPathParser.get_latest_lineage_summary(summary_dir) + self.assertEqual(None, result) + + args[0].return_value = [ + { + 'file_name': 'file0_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + } + ] + result = SummaryPathParser.get_latest_lineage_summary(summary_dir) + self.assertEqual(None, result) + + args[0].return_value = [ + { + 'file_name': 'file0_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file0_lineage_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file1_lineage', + 'create_time': datetime.fromtimestamp(1582031971) + }, + { + 'file_name': 'file1_lineage_lineage', + 'create_time': datetime.fromtimestamp(1582031971) + } + ] + result = SummaryPathParser.get_latest_lineage_summary(summary_dir) + self.assertEqual('/path/to/summary_dir/file1_lineage_lineage', result) + + @mock.patch.object(SummaryWatcher, 'list_summaries') + @mock.patch.object(SummaryWatcher, 'list_summary_directories') + def test_get_latest_lineage_summaries(self, *args): + """Test the function of get_latest_lineage_summaries.""" + args[0].return_value = [ + { + 'relative_path': './relative_path0' + }, + { + 'relative_path': './' + }, + { + 'relative_path': './relative_path1' + } + ] + args[1].return_value = [ + { + 'file_name': 'file0', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file0_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + }, + { + 'file_name': 'file1', + 'create_time': datetime.fromtimestamp(1582031971) + }, + { + 'file_name': 'file1_lineage', + 'create_time': datetime.fromtimestamp(1582031971) + } + ] + + expected_result = [ + '/path/to/base/relative_path0/file1_lineage', + '/path/to/base/file1_lineage', + '/path/to/base/relative_path1/file1_lineage' + ] + base_dir = '/path/to/base' + result = SummaryPathParser.get_latest_lineage_summaries(base_dir) + self.assertListEqual(expected_result, result) + + args[1].return_value = [ + { + 'file_name': 'file0_lineage', + 'create_time': datetime.fromtimestamp(1582031970) + } + ] + result = SummaryPathParser.get_latest_lineage_summaries(base_dir) + self.assertListEqual([], result) diff --git a/tests/ut/lineagemgr/common/validator/__init__.py b/tests/ut/lineagemgr/common/validator/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/lineagemgr/common/validator/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/common/validator/test_validate.py b/tests/ut/lineagemgr/common/validator/test_validate.py new file mode 100644 index 00000000..e8c06475 --- /dev/null +++ b/tests/ut/lineagemgr/common/validator/test_validate.py @@ -0,0 +1,283 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the validate module.""" +from unittest import TestCase + +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageParamValueError, LineageParamTypeError +from mindinsight.lineagemgr.common.validator.model_parameter import \ + SearchModelConditionParameter +from mindinsight.lineagemgr.common.validator.validate import \ + validate_search_model_condition +from mindinsight.utils.exceptions import MindInsightException + + +class TestValidateSearchModelCondition(TestCase): + """Test the mothod of validate_search_model_condition.""" + def test_validate_search_model_condition(self): + """Test the mothod of validate_search_model_condition.""" + condition = { + 'summary_dir': 'xxx' + } + self.assertRaisesRegex( + LineageParamTypeError, + 'The search_condition element summary_dir should be dict.', + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'xxx': 'xxx' + } + self.assertRaisesRegex( + LineageParamValueError, + 'The search attribute not supported.', + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'learning_rate': { + 'xxx': 'xxx' + } + } + self.assertRaisesRegex( + LineageParamValueError, + "The compare condition should be in", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + "offset": 100001 + } + self.assertRaisesRegex( + MindInsightException, + "Invalid input offset. 0 <= offset <= 100000", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'summary_dir': { + 'eq': 111 + }, + 'limit': 10 + } + self.assertRaisesRegex( + MindInsightException, + "The parameter summary_dir is invalid. It should be a dict and the value should be a string", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'learning_rate': { + 'in': 1.0 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'learning_rate': { + 'lt': True + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'learning_rate': { + 'gt': [1.0] + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'loss_function': { + 'ge': 1 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter loss_function is invalid. It should be a dict and the value should be a string", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'train_dataset_count': { + 'in': 2 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter train_dataset_count is invalid. It should be a dict " + "and the value should be a integer between 0", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'network': { + 'le': 2 + }, + 'optimizer': { + 'eq': 'xxx' + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter network is invalid. It should be a dict and the value should be a string", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'batch_size': { + 'lt': 2, + 'gt': 'xxx' + }, + 'model_size': { + 'eq': 222 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter batch_size is invalid. It should be a non-negative integer.", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'test_dataset_count': { + 'lt': -2 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter test_dataset_count is invalid. It should be a dict " + "and the value should be a integer between 0", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'epoch': { + 'lt': False + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter epoch is invalid. It should be a positive integer.", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + "learning_rate": { + "ge": "" + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter learning_rate is invalid. It should be a dict and the value should be a float or a integer", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + "train_dataset_count": { + "ge": 8.0 + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter train_dataset_count is invalid. It should be a dict " + "and the value should be a integer between 0", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 1: { + "ge": 8.0 + } + } + self.assertRaisesRegex( + LineageParamValueError, + "The search attribute not supported.", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'metric_': { + "ge": 8.0 + } + } + LineageParamValueError('The search attribute not supported.') + self.assertRaisesRegex( + LineageParamValueError, + "The search attribute not supported.", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) + + condition = { + 'metric_attribute': { + 'ge': 'xxx' + } + } + self.assertRaisesRegex( + MindInsightException, + "The parameter metric_attribute is invalid. " + "It should be a dict and the value should be a float or a integer", + validate_search_model_condition, + SearchModelConditionParameter, + condition + ) diff --git a/tests/ut/lineagemgr/querier/__init__.py b/tests/ut/lineagemgr/querier/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/lineagemgr/querier/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/querier/event_data.py b/tests/ut/lineagemgr/querier/event_data.py new file mode 100644 index 00000000..5cc5592b --- /dev/null +++ b/tests/ut/lineagemgr/querier/event_data.py @@ -0,0 +1,421 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The event data in querier test.""" +import json + +EVENT_TRAIN_DICT_0 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum0', + 'learning_rate': 0.10000000149011612, + 'loss_function': '', + 'epoch': 1, + 'parallel_mode': 'stand_alone0', + 'device_num': 1, + 'batch_size': 31 + }, + 'algorithm': { + 'network': 'TrainOneStepCell0', + 'loss': 2.3025848865509033 + }, + 'train_dataset': { + 'train_dataset_path': '', + 'train_dataset_size': 31 + }, + 'model': { + 'path': 'xxx0', + 'size': 400716930 + } + } +} + +EVENT_TRAIN_DICT_1 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum1', + 'learning_rate': 0.20000000298023224, + 'loss_function': 'loss_function1', + 'epoch': 1, + 'parallel_mode': 'stand_alone1', + 'device_num': 2, + 'batch_size': 35 + }, + 'algorithm': { + 'network': 'TrainOneStepCell1', + 'loss': 2.4025847911834717 + }, + 'train_dataset': { + 'train_dataset_path': '/path/to/train_dataset1', + 'train_dataset_size': 32 + }, + 'model': { + 'path': 'xxx1', + 'size': 400716931 + } + } +} + +EVENT_TRAIN_DICT_2 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum2', + 'learning_rate': 0.30000001192092896, + 'loss_function': 'loss_function2', + 'epoch': 2, + 'parallel_mode': 'stand_alone2', + 'device_num': 3, + 'batch_size': 38 + }, + 'algorithm': { + 'network': 'TrainOneStepCell2', + 'loss': 2.502584934234619 + }, + 'train_dataset': { + 'train_dataset_path': '/path/to/train_dataset2', + 'train_dataset_size': 33 + }, + 'model': { + 'path': 'xxx2', + 'size': 400716932 + } + } +} + +EVENT_TRAIN_DICT_3 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum3', + 'learning_rate': 0.4000000059604645, + 'loss_function': 'loss_function3', + 'epoch': 2, + 'parallel_mode': 'stand_alone3', + 'device_num': 3, + 'batch_size': 35 + }, + 'algorithm': { + 'network': 'TrainOneStepCell3', + 'loss': 2.6025848388671875 + }, + 'train_dataset': { + 'train_dataset_path': '/path/to/train_dataset3', + 'train_dataset_size': 34 + }, + 'model': { + 'path': 'xxx3', + 'size': 400716933 + } + } +} + +EVENT_TRAIN_DICT_4 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum4', + 'learning_rate': 0.5, + 'loss_function': 'loss_function1', + 'epoch': 3, + 'parallel_mode': 'stand_alone4', + 'device_num': 1, + 'batch_size': 50 + }, + 'algorithm': { + 'network': 'TrainOneStepCell4', + 'loss': 2.702584981918335 + }, + 'train_dataset': { + 'train_dataset_path': '/path/to/train_dataset4', + 'train_dataset_size': 35 + }, + 'model': { + 'path': 'xxx4', + 'size': 400716934 + } + } +} + +EVENT_TRAIN_DICT_5 = { + 'wall_time': 1581499557.7017336, + 'train_lineage': { + 'hyper_parameters': { + 'optimizer': 'ApplyMomentum5', + 'learning_rate': 0.5, + 'loss_function': 'loss_function1', + 'epoch': 3, + 'parallel_mode': 'stand_alone5', + 'device_num': 1, + 'batch_size': 51 + }, + 'algorithm': { + 'network': 'TrainOneStepCell5', + 'loss': 2.702584981918335 + }, + 'train_dataset': { + 'train_dataset_size': 35 + }, + 'model': { + 'path': 'xxx4', + 'size': 400716934 + } + } +} + +EVENT_TRAIN_DICT_EXCEPTION = { + 'wall_time': 1581499557.7017336 +} + +METRIC_0 = { + 'accuracy': None, + 'mae': 2.00000001, + 'mse': 3.00000001 +} + +METRIC_1 = { + 'accuracy': 1.0000002, + 'mae': 2.00000002, + 'mse': 3.00000002 +} + +METRIC_2 = { + 'accuracy': 1.0000003, + 'mae': 2.00000003, + 'mse': 3.00000003 +} + +METRIC_3 = { + 'accuracy': 1.0000004, + 'mae': 2.00000004, + 'mse': 3.00000004 +} + +METRIC_4 = { + 'accuracy': 1.0000005, + 'mae': 2.00000005, + 'mse': 3.00000005 +} + +METRIC_5 = { + 'accuracy': 1.0000006, + 'mae': 2.00000006, + 'mse': 3.00000006 +} + +EVENT_EVAL_DICT_0 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_0), + 'valid_dataset': { + 'valid_dataset_path': '', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_1 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_1), + 'valid_dataset': { + 'valid_dataset_path': '/path/to/valid_dataset1', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_2 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_2), + 'valid_dataset': { + 'valid_dataset_path': '/path/to/valid_dataset2', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_3 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_3), + 'valid_dataset': { + 'valid_dataset_path': '/path/to/valid_dataset3', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_4 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_4), + 'valid_dataset': { + 'valid_dataset_path': '/path/to/valid_dataset4', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_5 = { + 'wall_time': 1581499557.7017336, + 'evaluation_lineage': { + 'metric': json.dumps(METRIC_5), + 'valid_dataset': { + 'valid_dataset_path': '/path/to/valid_dataset5', + 'valid_dataset_size': 400716931 + } + } +} + +EVENT_EVAL_DICT_EXCEPTION = { + 'wall_time': 1581499557.7017336 +} + +EVENT_DATASET_DICT_0 = { + 'wall_time': 1583317727.4949381, + 'dataset_graph': { + 'children': [ + { + 'children': [ + { + 'parameter': { + 'mapStr': { + 'op_type': 'MnistDataset', + 'shard_id': 'None', + 'num_shards': 'None', + 'op_module': 'minddata.dataengine.datasets', + 'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData', + 'num_parallel_workers': 'None', + 'shuffle': 'None' + }, + 'mapInt': { + 'num_samples': 100 + } + }, + 'sampler': { + 'operationParam': { + 'mapStr': { + 'sampler_name': 'RandomSampler', + 'sampler_module': 'minddata.dataengine.samplers' + }, + 'mapBool': { + 'replacement': True + }, + 'mapInt': { + 'num_samples': 100 + } + } + } + } + ], + 'parameter': { + 'mapStr': { + 'op_module': 'minddata.dataengine.datasets', + 'op_type': 'MapDataset', + 'num_parallel_workers': 'None' + }, + 'mapStrList': { + 'output_columns': { + 'strValue': [ + '' + ] + }, + 'input_columns': { + 'strValue': [ + 'label' + ] + } + } + }, + 'operations': [ + { + 'operationParam': { + 'mapStr': { + 'tensor_op_module': 'minddata.transforms.c_transforms', + 'tensor_op_name': 'OneHot' + }, + 'mapInt': { + 'num_classes': 10 + } + } + } + ] + } + ], + 'parameter': { + 'mapStr': { + 'op_module': 'minddata.dataengine.datasets', + 'op_type': 'BatchDataset', + 'num_parallel_workers': 'None' + }, + 'mapBool': { + 'drop_remainder': True + }, + 'mapInt': { + 'batch_size': 10 + } + } + } +} + +DATASET_DICT_0 = { + 'op_type': 'BatchDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'drop_remainder': True, + 'batch_size': 10, + 'children': [ + { + 'op_type': 'MapDataset', + 'op_module': 'minddata.dataengine.datasets', + 'num_parallel_workers': None, + 'input_columns': [ + 'label' + ], + 'output_columns': [ + None + ], + 'operations': [ + { + 'tensor_op_module': 'minddata.transforms.c_transforms', + 'tensor_op_name': 'OneHot', + 'num_classes': 10 + } + ], + 'children': [ + { + 'op_type': 'MnistDataset', + 'shard_id': None, + 'num_shards': None, + 'op_module': 'minddata.dataengine.datasets', + 'dataset_dir': '/home/anthony/MindData/tests/dataset/data/testMnistData', + 'num_parallel_workers': None, + 'shuffle': None, + 'num_samples': 100, + 'sampler': { + 'sampler_module': 'minddata.dataengine.samplers', + 'sampler_name': 'RandomSampler', + 'replacement': True, + 'num_samples': 100 + }, + 'children': [] + } + ] + } + ] +} diff --git a/tests/ut/lineagemgr/querier/test_querier.py b/tests/ut/lineagemgr/querier/test_querier.py new file mode 100644 index 00000000..0eb39979 --- /dev/null +++ b/tests/ut/lineagemgr/querier/test_querier.py @@ -0,0 +1,860 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the querier module.""" +from unittest import TestCase, mock + +from google.protobuf.json_format import ParseDict + +import mindinsight.datavisual.proto_files.mindinsight_summary_pb2 as summary_pb2 +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageQuerierParamException, LineageParamTypeError, \ + LineageSummaryAnalyzeException, LineageSummaryParseException +from mindinsight.lineagemgr.querier.querier import Querier +from mindinsight.lineagemgr.summary.lineage_summary_analyzer import \ + LineageInfo +from . import event_data + + +def create_lineage_info(train_event_dict, eval_event_dict, dataset_event_dict): + """ + Create parsed lineage info tuple. + + Args: + train_event_dict (Union[dict, None]): The dict of train event. + eval_event_dict (Union[dict, None]): The dict of evaluation event. + dataset_event_dict (Union[dict, None]): The dict of dataset graph event. + + Returns: + namedtuple, parsed lineage info. + """ + if train_event_dict is not None: + train_event = summary_pb2.Event() + ParseDict(train_event_dict, train_event) + else: + train_event = None + + if eval_event_dict is not None: + eval_event = summary_pb2.Event() + ParseDict(eval_event_dict, eval_event) + else: + eval_event = None + + if dataset_event_dict is not None: + dataset_event = summary_pb2.Event() + ParseDict(dataset_event_dict, dataset_event) + else: + dataset_event = None + + lineage_info = LineageInfo( + train_lineage=train_event, + eval_lineage=eval_event, + dataset_graph=dataset_event, + ) + return lineage_info + + +def create_filtration_result(summary_dir, train_event_dict, + eval_event_dict, metric_dict, dataset_dict): + """ + Create filteration result. + + Args: + summary_dir (str): The summary dir. + train_event_dict (dict): The dict of train event. + eval_event_dict (dict): The dict of evaluation event. + metric_dict (dict): The dict of metric. + dataset_dict (dict): The dict of dataset graph. + + Returns: + dict, the filteration result. + """ + filtration_result = { + "summary_dir": summary_dir, + "loss_function": train_event_dict['train_lineage']['hyper_parameters']['loss_function'], + "train_dataset_path": train_event_dict['train_lineage']['train_dataset']['train_dataset_path'], + "train_dataset_count": train_event_dict['train_lineage']['train_dataset']['train_dataset_size'], + "test_dataset_path": eval_event_dict['evaluation_lineage']['valid_dataset']['valid_dataset_path'], + "test_dataset_count": eval_event_dict['evaluation_lineage']['valid_dataset']['valid_dataset_size'], + "network": train_event_dict['train_lineage']['algorithm']['network'], + "optimizer": train_event_dict['train_lineage']['hyper_parameters']['optimizer'], + "learning_rate": train_event_dict['train_lineage']['hyper_parameters']['learning_rate'], + "epoch": train_event_dict['train_lineage']['hyper_parameters']['epoch'], + "batch_size": train_event_dict['train_lineage']['hyper_parameters']['batch_size'], + "loss": train_event_dict['train_lineage']['algorithm']['loss'], + "model_size": train_event_dict['train_lineage']['model']['size'], + "metric": metric_dict, + "dataset_graph": dataset_dict, + "dataset_mark": '2', + } + return filtration_result + + +def get_lineage_infos(): + """ + Get tuples of lineage info, simulate the function of summary analyzer. + + Returns: + list[namedtuple], tuples of lineage info. + """ + train_events = [ + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_TRAIN_DICT_1, + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_TRAIN_DICT_5, + None + ] + eval_events = [ + event_data.EVENT_EVAL_DICT_0, + event_data.EVENT_EVAL_DICT_1, + event_data.EVENT_EVAL_DICT_2, + event_data.EVENT_EVAL_DICT_3, + event_data.EVENT_EVAL_DICT_4, + None, + event_data.EVENT_EVAL_DICT_5 + ] + dataset_events = [ + event_data.EVENT_DATASET_DICT_0 + ]*7 + + lineage_infos = list( + map( + lambda event: create_lineage_info(event[0], event[1], event[2]), + zip(train_events, eval_events, dataset_events) + ) + ) + + return lineage_infos + + +class TestQuerier(TestCase): + """Test the class of `Querier`.""" + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') + def setUp(self, *args): + """Initialization before test case execution.""" + args[0].return_value = create_lineage_info( + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.EVENT_DATASET_DICT_0 + ) + + single_summary_path = '/path/to/summary0/log0' + self.single_querier = Querier(single_summary_path) + + lineage_infos = get_lineage_infos() + args[0].side_effect = lineage_infos + summary_paths = [ + '/path/to/summary0/log0', + '/path/to/summary1/log1', + '/path/to/summary2/log2', + '/path/to/summary3/log3', + '/path/to/summary4/log4', + '/path/to/summary5/log5', + '/path/to/summary6/log6' + ] + self.multi_querier = Querier(summary_paths) + + def test_get_summary_lineage_success_1(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + **event_data.EVENT_TRAIN_DICT_0['train_lineage'], + 'metric': event_data.METRIC_0, + 'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = self.single_querier.get_summary_lineage() + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_success_2(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + **event_data.EVENT_TRAIN_DICT_0['train_lineage'], + 'metric': event_data.METRIC_0, + 'valid_dataset': + event_data.EVENT_EVAL_DICT_0['evaluation_lineage'][ + 'valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = self.single_querier.get_summary_lineage( + summary_dir='/path/to/summary0' + ) + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_success_3(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + 'model': event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'], + 'algorithm': event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm'] + } + ] + result = self.single_querier.get_summary_lineage( + filter_keys=['model', 'algorithm'] + ) + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_success_4(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + **event_data.EVENT_TRAIN_DICT_0['train_lineage'], + 'metric': event_data.METRIC_0, + 'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary1', + **event_data.EVENT_TRAIN_DICT_1['train_lineage'], + 'metric': event_data.METRIC_1, + 'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary2', + **event_data.EVENT_TRAIN_DICT_2['train_lineage'], + 'metric': event_data.METRIC_2, + 'valid_dataset': event_data.EVENT_EVAL_DICT_2['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary3', + **event_data.EVENT_TRAIN_DICT_3['train_lineage'], + 'metric': event_data.METRIC_3, + 'valid_dataset': event_data.EVENT_EVAL_DICT_3['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary4', + **event_data.EVENT_TRAIN_DICT_4['train_lineage'], + 'metric': event_data.METRIC_4, + 'valid_dataset': event_data.EVENT_EVAL_DICT_4['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary5', + **event_data.EVENT_TRAIN_DICT_5['train_lineage'], + 'metric': {}, + 'valid_dataset': {}, + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary6', + 'hyper_parameters': {}, + 'algorithm': {}, + 'model': {}, + 'train_dataset': {}, + 'metric': event_data.METRIC_5, + 'valid_dataset': event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = self.multi_querier.get_summary_lineage() + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_success_5(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary1', + **event_data.EVENT_TRAIN_DICT_1['train_lineage'], + 'metric': event_data.METRIC_1, + 'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = self.multi_querier.get_summary_lineage( + summary_dir='/path/to/summary1' + ) + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_success_6(self): + """Test the success of get_summary_lineage.""" + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + 'hyper_parameters': event_data.EVENT_TRAIN_DICT_0['train_lineage']['hyper_parameters'], + 'train_dataset': event_data.EVENT_TRAIN_DICT_0['train_lineage']['train_dataset'], + 'metric': event_data.METRIC_0, + 'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'] + } + ] + filter_keys = [ + 'metric', 'hyper_parameters', 'train_dataset', 'valid_dataset' + ] + result = self.multi_querier.get_summary_lineage( + summary_dir='/path/to/summary0', filter_keys=filter_keys + ) + self.assertListEqual(expected_result, result) + + def test_get_summary_lineage_fail(self): + """Test the function of get_summary_lineage with exception.""" + filter_keys = ['xxx'] + self.assertRaises( + LineageQuerierParamException, + self.multi_querier.get_summary_lineage, + filter_keys=filter_keys + ) + + self.assertRaises( + LineageQuerierParamException, + self.multi_querier.get_summary_lineage, + summary_dir='xxx' + ) + + def test_filter_summary_lineage_success_1(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'optimizer': { + 'in': [ + 'ApplyMomentum0', + 'ApplyMomentum1', + 'ApplyMomentum2', + 'ApplyMomentum4' + ] + }, + 'learning_rate': { + 'lt': 0.5, + 'gt': 0.2 + }, + 'sorted_name': 'summary_dir' + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary1', + event_data.EVENT_TRAIN_DICT_1, + event_data.EVENT_EVAL_DICT_1, + event_data.METRIC_1, + event_data.DATASET_DICT_0, + ), + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ) + ], + 'count': 2, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_2(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'batch_size': { + 'le': 50, + 'ge': 35 + }, + 'model_size': { + 'lt': 400716934, + 'gt': 400716931 + }, + 'sorted_name': 'batch_size', + 'sorted_type': 'descending' + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary3', + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_EVAL_DICT_3, + event_data.METRIC_3, + event_data.DATASET_DICT_0 + ) + ], + 'count': 2, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_3(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'limit': 2, + 'offset': 1 + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary3', + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_EVAL_DICT_3, + event_data.METRIC_3, + event_data.DATASET_DICT_0 + ) + ], + 'count': 7, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_4(self): + """Test the success of filter_summary_lineage.""" + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary0', + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.METRIC_0, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary1', + event_data.EVENT_TRAIN_DICT_1, + event_data.EVENT_EVAL_DICT_1, + event_data.METRIC_1, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary3', + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_EVAL_DICT_3, + event_data.METRIC_3, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary4', + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_EVAL_DICT_4, + event_data.METRIC_4, + event_data.DATASET_DICT_0 + ), + { + "summary_dir": '/path/to/summary5', + "loss_function": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'], + "train_dataset_path": None, + "train_dataset_count": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'], + "test_dataset_path": None, + "test_dataset_count": None, + "network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'], + "optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'], + "learning_rate": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'], + "epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'], + "batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'], + "loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'], + "model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'], + "metric": {}, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + }, + { + "summary_dir": '/path/to/summary6', + "loss_function": None, + "train_dataset_path": None, + "train_dataset_count": None, + "test_dataset_path": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'], + "test_dataset_count": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'], + "network": None, + "optimizer": None, + "learning_rate": None, + "epoch": None, + "batch_size": None, + "loss": None, + "model_size": None, + "metric": event_data.METRIC_5, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + } + ], + 'count': 7, + } + result = self.multi_querier.filter_summary_lineage() + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_5(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'optimizer': { + 'eq': 'ApplyMomentum4' + } + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary4', + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_EVAL_DICT_4, + event_data.METRIC_4, + event_data.DATASET_DICT_0 + ), + ], + 'count': 1, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_6(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'sorted_name': 'metric_accuracy', + 'sorted_type': 'ascending' + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary0', + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.METRIC_0, + event_data.DATASET_DICT_0 + ), + { + "summary_dir": '/path/to/summary5', + "loss_function": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'], + "train_dataset_path": None, + "train_dataset_count": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'], + "test_dataset_path": None, + "test_dataset_count": None, + "network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'], + "optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'], + "learning_rate": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'], + "epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'], + "batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'], + "loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'], + "model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'], + "metric": {}, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + }, + create_filtration_result( + '/path/to/summary1', + event_data.EVENT_TRAIN_DICT_1, + event_data.EVENT_EVAL_DICT_1, + event_data.METRIC_1, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary3', + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_EVAL_DICT_3, + event_data.METRIC_3, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary4', + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_EVAL_DICT_4, + event_data.METRIC_4, + event_data.DATASET_DICT_0 + ), + { + "summary_dir": '/path/to/summary6', + "loss_function": None, + "train_dataset_path": None, + "train_dataset_count": None, + "test_dataset_path": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'], + "test_dataset_count": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'], + "network": None, + "optimizer": None, + "learning_rate": None, + "epoch": None, + "batch_size": None, + "loss": None, + "model_size": None, + "metric": event_data.METRIC_5, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + } + ], + 'count': 7, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_7(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'sorted_name': 'metric_accuracy', + 'sorted_type': 'descending' + } + expected_result = { + 'object': [ + { + "summary_dir": '/path/to/summary6', + "loss_function": None, + "train_dataset_path": None, + "train_dataset_count": None, + "test_dataset_path": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_path'], + "test_dataset_count": + event_data.EVENT_EVAL_DICT_5['evaluation_lineage']['valid_dataset']['valid_dataset_size'], + "network": None, + "optimizer": None, + "learning_rate": None, + "epoch": None, + "batch_size": None, + "loss": None, + "model_size": None, + "metric": event_data.METRIC_5, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + }, + create_filtration_result( + '/path/to/summary4', + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_EVAL_DICT_4, + event_data.METRIC_4, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary3', + event_data.EVENT_TRAIN_DICT_3, + event_data.EVENT_EVAL_DICT_3, + event_data.METRIC_3, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary2', + event_data.EVENT_TRAIN_DICT_2, + event_data.EVENT_EVAL_DICT_2, + event_data.METRIC_2, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary1', + event_data.EVENT_TRAIN_DICT_1, + event_data.EVENT_EVAL_DICT_1, + event_data.METRIC_1, + event_data.DATASET_DICT_0 + ), + create_filtration_result( + '/path/to/summary0', + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.METRIC_0, + event_data.DATASET_DICT_0 + ), + { + "summary_dir": '/path/to/summary5', + "loss_function": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['loss_function'], + "train_dataset_path": None, + "train_dataset_count": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['train_dataset']['train_dataset_size'], + "test_dataset_path": None, + "test_dataset_count": None, + "network": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['network'], + "optimizer": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['optimizer'], + "learning_rate": + event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['learning_rate'], + "epoch": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['epoch'], + "batch_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['hyper_parameters']['batch_size'], + "loss": event_data.EVENT_TRAIN_DICT_5['train_lineage']['algorithm']['loss'], + "model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'], + "metric": {}, + "dataset_graph": event_data.DATASET_DICT_0, + "dataset_mark": '2' + } + ], + 'count': 7, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_8(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'metric_accuracy': { + 'lt': 1.0000006, + 'gt': 1.0000004 + } + } + expected_result = { + 'object': [ + create_filtration_result( + '/path/to/summary4', + event_data.EVENT_TRAIN_DICT_4, + event_data.EVENT_EVAL_DICT_4, + event_data.METRIC_4, + event_data.DATASET_DICT_0 + ), + ], + 'count': 1, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_success_9(self): + """Test the success of filter_summary_lineage.""" + condition = { + 'limit': 3, + 'offset': 3 + } + expected_result = { + 'object': [], + 'count': 7, + } + result = self.multi_querier.filter_summary_lineage(condition=condition) + self.assertDictEqual(expected_result, result) + + def test_filter_summary_lineage_fail(self): + """Test the function of filter_summary_lineage with exception.""" + condition = { + 'xxx': { + 'lt': 1.0000006, + 'gt': 1.0000004 + } + } + self.assertRaises( + LineageQuerierParamException, + self.multi_querier.filter_summary_lineage, + condition=condition + ) + + condition = { + 'accuracy': { + 'xxx': 1 + } + } + self.assertRaises( + LineageQuerierParamException, + self.multi_querier.filter_summary_lineage, + condition=condition + ) + + condition = { + 'sorted_name': 'xxx' + } + self.assertRaises( + LineageQuerierParamException, + self.multi_querier.filter_summary_lineage, + condition=condition + ) + + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') + def test_init_fail(self, *args): + """Test the function of init with exception.""" + summary_path = {'xxx': 1} + with self.assertRaises(LineageParamTypeError): + Querier(summary_path) + + summary_path = None + with self.assertRaises(LineageQuerierParamException): + Querier(summary_path) + + args[0].side_effect = LineageSummaryAnalyzeException + summary_path = '/path/to/summary0/log0' + with self.assertRaises(LineageSummaryParseException): + Querier(summary_path) + + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') + def test_parse_fail_summary_logs_1(self, *args): + """Test the function of parsing fail summary logs.""" + lineage_infos = get_lineage_infos() + args[0].side_effect = lineage_infos + + summary_path = ['/path/to/summary0/log0'] + querier = Querier(summary_path) + querier._parse_failed_paths.append('/path/to/summary1/log1') + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + **event_data.EVENT_TRAIN_DICT_0['train_lineage'], + 'metric': event_data.METRIC_0, + 'valid_dataset': event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + }, + { + 'summary_dir': '/path/to/summary1', + **event_data.EVENT_TRAIN_DICT_1['train_lineage'], + 'metric': event_data.METRIC_1, + 'valid_dataset': event_data.EVENT_EVAL_DICT_1['evaluation_lineage']['valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = querier.get_summary_lineage() + self.assertListEqual(expected_result, result) + self.assertListEqual([], querier._parse_failed_paths) + + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') + def test_parse_fail_summary_logs_2(self, *args): + """Test the function of parsing fail summary logs.""" + args[0].return_value = create_lineage_info( + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.EVENT_DATASET_DICT_0, + ) + + summary_path = ['/path/to/summary0/log0'] + querier = Querier(summary_path) + querier._parse_failed_paths.append('/path/to/summary1/log1') + + args[0].return_value = create_lineage_info(None, None, None) + expected_result = [ + { + 'summary_dir': '/path/to/summary0', + **event_data.EVENT_TRAIN_DICT_0['train_lineage'], + 'metric': event_data.METRIC_0, + 'valid_dataset': + event_data.EVENT_EVAL_DICT_0['evaluation_lineage'][ + 'valid_dataset'], + 'dataset_graph': event_data.DATASET_DICT_0 + } + ] + result = querier.get_summary_lineage() + self.assertListEqual(expected_result, result) + self.assertListEqual( + ['/path/to/summary1/log1'], querier._parse_failed_paths + ) diff --git a/tests/ut/lineagemgr/querier/test_query_model.py b/tests/ut/lineagemgr/querier/test_query_model.py new file mode 100644 index 00000000..12d566f7 --- /dev/null +++ b/tests/ut/lineagemgr/querier/test_query_model.py @@ -0,0 +1,156 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the query_model module.""" +from unittest import TestCase + +from mindinsight.lineagemgr.common.exceptions.exceptions import \ + LineageEventNotExistException, LineageEventFieldNotExistException +from mindinsight.lineagemgr.querier.query_model import LineageObj +from . import event_data +from .test_querier import create_lineage_info, create_filtration_result + + +class TestLineageObj(TestCase): + """Test the class of `LineageObj`.""" + + def setUp(self): + """Initialization before test case execution.""" + lineage_info = create_lineage_info( + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.EVENT_DATASET_DICT_0 + ) + self.summary_dir = '/path/to/summary0' + self.lineage_obj = LineageObj( + self.summary_dir, + train_lineage=lineage_info.train_lineage, + evaluation_lineage=lineage_info.eval_lineage, + dataset_graph=lineage_info.dataset_graph, + ) + + lineage_info = create_lineage_info( + event_data.EVENT_TRAIN_DICT_0, + None, None) + self.lineage_obj_no_eval = LineageObj( + self.summary_dir, + train_lineage=lineage_info.train_lineage, + evaluation_lineage=lineage_info.eval_lineage + ) + + def test_property(self): + """Test the function of getting property.""" + self.assertEqual(self.summary_dir, self.lineage_obj.summary_dir) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm'], + self.lineage_obj.algorithm + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'], + self.lineage_obj.model + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['train_dataset'], + self.lineage_obj.train_dataset + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['hyper_parameters'], + self.lineage_obj.hyper_parameters + ) + self.assertDictEqual(event_data.METRIC_0, self.lineage_obj.metric) + self.assertDictEqual( + event_data.EVENT_EVAL_DICT_0['evaluation_lineage']['valid_dataset'], + self.lineage_obj.valid_dataset + ) + + def test_property_eval_not_exist(self): + """Test the function of getting property with no evaluation event.""" + self.assertEqual(self.summary_dir, self.lineage_obj.summary_dir) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm'], + self.lineage_obj_no_eval.algorithm + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'], + self.lineage_obj_no_eval.model + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['train_dataset'], + self.lineage_obj_no_eval.train_dataset + ) + self.assertDictEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['hyper_parameters'], + self.lineage_obj_no_eval.hyper_parameters + ) + self.assertDictEqual({}, self.lineage_obj_no_eval.metric) + self.assertDictEqual({}, self.lineage_obj_no_eval.valid_dataset) + + def test_get_summary_info(self): + """Test the function of get_summary_info.""" + filter_keys = ['algorithm', 'model'] + expected_result = { + 'summary_dir': self.summary_dir, + 'algorithm': event_data.EVENT_TRAIN_DICT_0['train_lineage']['algorithm'], + 'model': event_data.EVENT_TRAIN_DICT_0['train_lineage']['model'] + } + result = self.lineage_obj.get_summary_info(filter_keys) + self.assertDictEqual(expected_result, result) + + def test_to_filtration_dict(self): + """Test the function of to_filtration_dict.""" + expected_result = create_filtration_result( + self.summary_dir, + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_0, + event_data.METRIC_0, + event_data.DATASET_DICT_0 + ) + expected_result['dataset_mark'] = None + result = self.lineage_obj.to_filtration_dict() + self.assertDictEqual(expected_result, result) + + def test_get_value_by_key(self): + """Test the function of get_value_by_key.""" + result = self.lineage_obj.get_value_by_key('model_size') + self.assertEqual( + event_data.EVENT_TRAIN_DICT_0['train_lineage']['model']['size'], + result + ) + + def test_init_fail(self): + """Test the function of init with exception.""" + with self.assertRaises(LineageEventNotExistException): + LineageObj(self.summary_dir) + + lineage_info = create_lineage_info( + event_data.EVENT_TRAIN_DICT_EXCEPTION, None, None + ) + with self.assertRaises(LineageEventFieldNotExistException): + self.lineage_obj = LineageObj( + self.summary_dir, + train_lineage=lineage_info.train_lineage, + evaluation_lineage=lineage_info.eval_lineage + ) + + lineage_info = create_lineage_info( + event_data.EVENT_TRAIN_DICT_0, + event_data.EVENT_EVAL_DICT_EXCEPTION, + event_data.EVENT_DATASET_DICT_0 + ) + with self.assertRaises(LineageEventFieldNotExistException): + self.lineage_obj = LineageObj( + self.summary_dir, + train_lineage=lineage_info.train_lineage, + evaluation_lineage=lineage_info.eval_lineage + ) diff --git a/tests/ut/lineagemgr/summary/__init__.py b/tests/ut/lineagemgr/summary/__init__.py new file mode 100644 index 00000000..e3077430 --- /dev/null +++ b/tests/ut/lineagemgr/summary/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/lineagemgr/summary/test_event_writer.py b/tests/ut/lineagemgr/summary/test_event_writer.py new file mode 100644 index 00000000..d3f1bda9 --- /dev/null +++ b/tests/ut/lineagemgr/summary/test_event_writer.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test class EventWriter.""" +import os +from unittest import TestCase + +from mindinsight.lineagemgr.summary.event_writer import EventWriter +from mindinsight.lineagemgr.summary.summary_record import LineageSummary +from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageSummaryAnalyzer + + +class TestEventWriter(TestCase): + """Test write_event_to_file.""" + def setUp(self): + """The setup of test.""" + self.log_path = "./test.log" + + def test_write_event_to_file(self): + """Test write event to file.""" + run_context_args = {"train_network": "res"} + content = LineageSummary.package_train_message(run_context_args).SerializeToString() + event_writer = EventWriter(self.log_path, True) + event_writer.write_event_to_file(content) + + lineage_info = LineageSummaryAnalyzer.get_summary_infos(self.log_path) + self.assertEqual( + lineage_info.train_lineage.train_lineage.algorithm.network, + run_context_args.get("train_network") + ) + + def tearDown(self): + """The setup of test.""" + if os.path.exists(self.log_path): + try: + os.remove(self.log_path) + except IOError: + pass diff --git a/tests/ut/lineagemgr/summary/test_file_handler.py b/tests/ut/lineagemgr/summary/test_file_handler.py new file mode 100644 index 00000000..7dc73e0c --- /dev/null +++ b/tests/ut/lineagemgr/summary/test_file_handler.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test file_handler.py.""" +from unittest import mock, TestCase + +from mindinsight.lineagemgr.summary.file_handler import FileHandler + + +class TestFileHandler(TestCase): + """Test file_handler.py""" + + @mock.patch("os.path.getsize", return_value=12) + @mock.patch("builtins.open") + def setUp(self, *args): + args[0].return_value.__enter__.return_value.read.return_value = b'\x0a\x0b\x0c' * 4 + self.file_handler = FileHandler("fake_path.log") + + def test_seek(self): + """Test seek method.""" + self.file_handler.seek(5) + cur_pos = self.file_handler.tell() + self.assertEqual(cur_pos, 5) + + def test_read(self): + """Test read method.""" + res = self.file_handler.read(3) + self.assertEqual(res, b'\x0a\x0b\x0c') + + def test_read_with_pos(self): + """Test read method with specific position.""" + res = self.file_handler.read(3, 1) + self.assertEqual(res, b'\x0b\x0c\x0a') + + def test_size(self): + """Test size property.""" + size = self.file_handler.size + self.assertEqual(size, 12) diff --git a/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py b/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py new file mode 100644 index 00000000..96e511be --- /dev/null +++ b/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py @@ -0,0 +1,142 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test LineageSummaryAnalyzer.py.""" +from unittest import mock, TestCase +from unittest.mock import MagicMock + +from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageVerificationException, \ + LineageSummaryAnalyzeException +from mindinsight.lineagemgr.common.log import logger as log +from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageSummaryAnalyzer, \ + LineageInfo, SummaryAnalyzer + + +class TestSummaryAnalyzer(TestCase): + """Test SummaryAnalyzer class.""" + + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.FileHandler") + def setUp(self, *args): + args[0].return_value = MagicMock() + self.analyzer = SummaryAnalyzer("fake_path.log") + + @mock.patch.object(SummaryAnalyzer, '_has_next') + @mock.patch.object(SummaryAnalyzer, '_read_event') + def test_load_events(self, *args): + """Test load_events method.""" + args[1].side_effect = [True, False] + args[0].return_value = "event" + res = self.analyzer.load_events() + self.assertEqual(list(res), ["event"]) + + def test_has_next_true(self): + """Test has_next method when return value is True.""" + self.analyzer.file_handler.tell.return_value = 0 + self.analyzer.file_handler.size = 10 + res = self.analyzer._has_next() + self.assertEqual(res, True) + + def test_has_next_false(self): + """Test has_next method when return value is False.""" + self.analyzer.file_handler.tell.return_value = 10 + self.analyzer.file_handler.size = 10 + res = self.analyzer._has_next() + self.assertEqual(res, False) + + @mock.patch.object(SummaryAnalyzer, '_read_header') + @mock.patch.object(SummaryAnalyzer, '_read_body') + @mock.patch.object(Event, 'FromString') + def test_read_event(self, *args): + """Test read_event method.""" + args[2].return_value = 10 + args[1].return_value = "a" * 10 + args[0].return_value = "event" + res = self.analyzer._read_event() + self.assertEqual(res, "event") + + @mock.patch.object(SummaryAnalyzer, '_check_crc') + def test_read_header(self, *args): + """Test read_header method.""" + args[0].return_value = None + header_str = b'\x01' + b'\x00' * 7 + header_crc_str = b'\x00' * 4 + self.analyzer.file_handler.read.side_effect = [ + header_str, header_crc_str] + res = self.analyzer._read_header() + self.assertEqual(res, 1) + + @mock.patch.object(SummaryAnalyzer, '_check_crc') + def test_read_body(self, *args): + """Test read_body method.""" + args[0].return_value = None + body_str = b'\x01' * 5 + body_crc_str = b'\x00' * 4 + self.analyzer.file_handler.read.side_effect = [ + body_str, body_crc_str] + res = self.analyzer._read_body(5) + self.assertEqual(res, body_str) + + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.crc32") + @mock.patch.object(log, "error") + def test_check_crc(self, *args): + """Test _check_crc method.""" + args[0].return_value = None + args[1].GetValueFromStr.return_value = 1 + args[1].GetMaskCrc32cValue.return_value = 2 + source_str = b'\x01' * 10 + crc_str = b'\x00' * 4 + with self.assertRaisesRegex(LineageVerificationException, "The CRC verification failed."): + self.analyzer._check_crc(source_str, crc_str) + + +class TestLineageSummaryAnalyzer(TestCase): + """Test LineageSummaryAnalyzer class.""" + + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.safe_normalize_path") + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.FileHandler") + def setUp(self, *args): + args[0].return_value = MagicMock() + self.analyzer = LineageSummaryAnalyzer("fake_path.log") + + @mock.patch.object(LineageSummaryAnalyzer, 'load_events') + def test_get_latest_info(self, *args): + """Test get_latest_info method.""" + mock_event = MagicMock() + mock_event.HasField.side_effect = [False, True] + args[0].return_value = [mock_event] + res = self.analyzer.get_latest_info() + self.assertEqual(res.train_lineage, None) + self.assertEqual(res.eval_lineage, mock_event) + + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.safe_normalize_path") + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.FileHandler") + @mock.patch.object(LineageSummaryAnalyzer, 'get_latest_info') + def test_get_summary_infos(self, *args): + """Test get_summary_infos method.""" + args[0].return_value = LineageInfo(None, None, None) + res = LineageSummaryAnalyzer.get_summary_infos('fake_path.log') + self.assertEqual(res.train_lineage, None) + self.assertEqual(res.eval_lineage, None) + + @mock.patch.object(log, "error") + @mock.patch.object(log, "exception") + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.safe_normalize_path") + @mock.patch("mindinsight.lineagemgr.summary.lineage_summary_analyzer.FileHandler") + @mock.patch.object(LineageSummaryAnalyzer, 'get_latest_info') + def test_get_summary_infos_except(self, *args): + """Test get_summary_infos method with exception.""" + args[0].side_effect = LineageVerificationException("mock exception") + with self.assertRaises(LineageSummaryAnalyzeException): + _ = LineageSummaryAnalyzer.get_summary_infos('fake_path.log') diff --git a/tests/ut/lineagemgr/summary/test_summary_record.py b/tests/ut/lineagemgr/summary/test_summary_record.py new file mode 100644 index 00000000..15b5f69e --- /dev/null +++ b/tests/ut/lineagemgr/summary/test_summary_record.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test class SummaryRecord.""" +import json +from unittest import TestCase, mock + +from mindinsight.lineagemgr.summary.event_writer import EventWriter +from mindinsight.lineagemgr.summary.summary_record import LineageSummary + + +class TestSummaryRecord(TestCase): + """Test summary record.""" + def setUp(self): + """The setup of test.""" + self.run_context_args = dict() + self.run_context_args["train_network"] = "test_train_network" + self.run_context_args["loss"] = 0.1 + self.run_context_args["learning_rate"] = 0.1 + self.run_context_args["optimizer"] = "test_optimizer" + self.run_context_args["loss_function"] = "test_loss_function" + self.run_context_args["epoch"] = 1 + self.run_context_args["parallel_mode"] = "test_parallel_mode" + self.run_context_args["device_num"] = 1 + self.run_context_args["batch_size"] = 1 + self.run_context_args["train_dataset_path"] = "test_train_dataset_path" + self.run_context_args["train_dataset_size"] = 1 + self.run_context_args["model_path"] = "test_model_path" + self.run_context_args["model_size"] = 1 + + self.eval_args = dict() + self.eval_args["metrics"] = json.dumps({"acc": "test"}) + self.eval_args["valid_dataset_path"] = "test_valid_dataset_path" + self.eval_args["valid_dataset_size"] = 1 + + self.hard_info_args = dict() + self.hard_info_args["pid"] = 1 + self.hard_info_args["process_start_time"] = 921226.0 + + def test_package_train_message(self): + """Test package_train_message.""" + event = LineageSummary.package_train_message(self.run_context_args) + self.assertEqual( + event.train_lineage.algorithm.network, self.run_context_args.get("train_network")) + self.assertEqual( + event.train_lineage.hyper_parameters.optimizer, self.run_context_args.get("optimizer")) + self.assertEqual( + event.train_lineage.train_dataset.train_dataset_path, + self.run_context_args.get("train_dataset_path") + ) + + @mock.patch.object(EventWriter, "write_event_to_file") + def test_record_train_lineage(self, write_file): + """Test record_train_lineage.""" + write_file.return_value = True + lineage_summray = LineageSummary(summary_log_path="test.log") + lineage_summray.record_train_lineage(self.run_context_args) + + def test_package_evaluation_message(self): + """Test package_evaluation_message.""" + event = LineageSummary.package_evaluation_message(self.eval_args) + self.assertEqual(event.evaluation_lineage.metric, self.eval_args.get("metrics")) + + @mock.patch.object(EventWriter, "write_event_to_file") + def test_record_eval_lineage(self, write_file): + """Test record_eval_lineage.""" + write_file.return_value = True + lineage_summray = LineageSummary(summary_log_path="test.log") + lineage_summray.record_evaluation_lineage(self.eval_args) diff --git a/tests/ut/runtest.sh b/tests/ut/runtest.sh new file mode 100644 index 00000000..ab96ab9d --- /dev/null +++ b/tests/ut/runtest.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +shopt -s nullglob + +SCRIPT_BASEDIR=$( + cd "$(dirname "$0")" || exit + pwd +) +PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") +CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh" +CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/" +UT_PATH="${PROJECT_DIR}/tests/ut" +IS_BUILD_CRC="" + +build_crc32() { + echo "Start to check crc32." + if [ -d "$CRC32_OUTPUT_DIR" ]; then + cd "$CRC32_OUTPUT_DIR" + result=$(find . -maxdepth 1 -name "crc32*.so") + if [ -z "$result" ]; then + echo "Start to build crc32." + IS_BUILD_CRC="true" + bash "$CRC32_SCRIPT_PATH" + fi + fi + +} + +clean_crc32() { + echo "Start to clean crc32." + if [ -n "$IS_BUILD_CRC" ]; then + rm "$CRC32_OUTPUT_DIR"/crc32*.so -f + fi +} + +before_run_test() { + echo "Before run tests." + export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH + build_crc32 +} + +after_run_test() { + echo "After run tests." + clean_crc32 + + echo "End to run test." +} + +run_test() { + echo "Start to run test." + cd "$PROJECT_DIR" + + pytest "$UT_PATH" + + echo "Test all use cases success." +} + +before_run_test +run_test +after_run_test diff --git a/third_party/securec/CMakeLists.txt b/third_party/securec/CMakeLists.txt new file mode 100644 index 00000000..b304ead6 --- /dev/null +++ b/third_party/securec/CMakeLists.txt @@ -0,0 +1,11 @@ +SET(CMAKE_BUILD_TYPE "Debug") +SET(CMAKE_C_FLAGS_DEBUG "$ENV{CFLAGS} -fPIC -O0 -Wall -fvisibility=hidden -Wno-deprecated-declarations -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer -D_LIBCPP_INLINE_VISIBILITY='' -D'_LIBCPP_EXTERN_TEMPLATE(...)='") +SET(CMAKE_C_FLAGS_RELEASE "$ENV{CFLAGS} -fPIC -O3 -Wall -fvisibility=hidden -Wno-deprecated-declarations") +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +#add flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I/usr/local/include -Werror") + + +include_directories(./include) +add_subdirectory(src) diff --git a/third_party/securec/include/securec.h b/third_party/securec/include/securec.h new file mode 100644 index 00000000..0165583b --- /dev/null +++ b/third_party/securec/include/securec.h @@ -0,0 +1,633 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 +#define __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 + +#include "securectype.h" +#include + +#ifndef SECUREC_HAVE_ERRNO_H +#if SECUREC_IN_KERNEL +#define SECUREC_HAVE_ERRNO_H 0 +#else +#define SECUREC_HAVE_ERRNO_H 1 +#endif +#endif + +/* EINVAL ERANGE may defined in errno.h */ +#if SECUREC_HAVE_ERRNO_H +#include +#endif + +/* define error code */ +#if defined(SECUREC_NEED_ERRNO_TYPE) || !defined(__STDC_WANT_LIB_EXT1__) || \ + (defined(__STDC_WANT_LIB_EXT1__) && (__STDC_WANT_LIB_EXT1__ == 0)) +#ifndef SECUREC_DEFINED_ERRNO_TYPE +#define SECUREC_DEFINED_ERRNO_TYPE +/* just check whether macrodefinition exists. */ +#ifndef errno_t +typedef int errno_t; +#endif +#endif +#endif + +/* success */ +#ifndef EOK +#define EOK 0 +#endif + +#ifndef EINVAL +/* The src buffer is not correct and destination buffer cant not be reset */ +#define EINVAL 22 +#endif + +#ifndef EINVAL_AND_RESET +/* Once the error is detected, the dest buffer must be reseted! */ +#define EINVAL_AND_RESET (22 | 128) +#endif + +#ifndef ERANGE +/* The destination buffer is not long enough and destination buffer can not be reset */ +#define ERANGE 34 +#endif + +#ifndef ERANGE_AND_RESET +/* Once the error is detected, the dest buffer must be reseted! */ +#define ERANGE_AND_RESET (34 | 128) +#endif + +#ifndef EOVERLAP_AND_RESET +/* Once the buffer overlap is detected, the dest buffer must be reseted! */ +#define EOVERLAP_AND_RESET (54 | 128) +#endif + +/* if you need export the function of this library in Win32 dll, use __declspec(dllexport) */ +#ifndef SECUREC_API +#if defined(SECUREC_DLL_EXPORT) +#define SECUREC_API __declspec(dllexport) +#elif defined(SECUREC_DLL_IMPORT) +#define SECUREC_API __declspec(dllimport) +#else +/* Standardized function declaration . If a security function is declared in the your code, + * it may cause a compilation alarm,Please delete the security function you declared + * Adding extern under windows will cause the system to have inline functions to expand, + * so do not add the extern in default + */ +#if defined(_MSC_VER) +#define SECUREC_API +#else +#define SECUREC_API extern +#endif +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + /* + * Description: The GetHwSecureCVersion function get SecureC Version string and version number. + * Parameter: verNumber - to store version number + * Return: version string + */ + SECUREC_API const char *GetHwSecureCVersion(unsigned short *verNumber); + +#if SECUREC_ENABLE_MEMSET + /* + * Description: The memset_s function copies the value of c (converted to an unsigned char) into each of + * the first count characters of the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: c - the value to be copied + * Parameter: count -copies fisrt count characters of dest + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memset_s(void *dest, size_t destMax, int c, size_t count); +#endif + +#ifndef SECUREC_ONLY_DECLARE_MEMSET +#define SECUREC_ONLY_DECLARE_MEMSET 0 +#endif + +#if SECUREC_ONLY_DECLARE_MEMSET == 0 + +#if SECUREC_ENABLE_MEMMOVE + /* + * Description: The memmove_s function copies n characters from the object pointed to by src + * into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memmove_s(void *dest, size_t destMax, const void *src, size_t count); +#endif + +#if SECUREC_ENABLE_MEMCPY + /* + * Description: The memcpy_s function copies n characters from the object pointed to + * by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count); +#endif + +#if SECUREC_ENABLE_STRCPY + /* + * Description: The strcpy_s function copies the string pointed to by strSrc (including + * the terminating null character) into the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strcpy_s(char *strDest, size_t destMax, const char *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCPY + /* + * Description: The strncpy_s function copies not more than n successive characters (not including + * the terminating null character) + * from the array pointed to by strSrc to the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strncpy_s(char *strDest, size_t destMax, const char *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRCAT + /* + * Description: The strcat_s function appends a copy of the string pointed to by strSrc (including + * the terminating null character) + * to the end of the string pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null wide character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strcat_s(char *strDest, size_t destMax, const char *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCAT + /* + * Description: The strncat_s function appends not more than n successive characters (not including + * the terminating null character) + * from the array pointed to by strSrc to the end of the string pointed to by strDest. + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Parameter: strSrc -source address + * Parameter: count -copies count characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t strncat_s(char *strDest, size_t destMax, const char *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_VSPRINTF + /* + * Description: The vsprintf_s function is equivalent to the vsprintf function except for the Parameter: destMax + * and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null wide characte) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1. + */ + SECUREC_API int vsprintf_s(char *strDest, size_t destMax, const char *format, + va_list argList) SECUREC_ATTRIBUTE(3, 0); +#endif + +#if SECUREC_ENABLE_SPRINTF + /* + * Description: The sprintf_s function is equivalent to the sprintf function except for the Parameter: destMax + * and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1. + */ + SECUREC_API int sprintf_s(char *strDest, size_t destMax, const char *format, ...) SECUREC_ATTRIBUTE(3, 4); +#endif + +#if SECUREC_ENABLE_VSNPRINTF + /* + * Description: The vsnprintf_s function is equivalent to the vsnprintf function except for the Parameter: + * destMax/count and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: count - do not write more than count bytes to strDest(not including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning -1 when truncation occurs + */ + SECUREC_API int vsnprintf_s(char *strDest, size_t destMax, size_t count, const char *format, + va_list argList) SECUREC_ATTRIBUTE(4, 0); +#endif + +#if SECUREC_ENABLE_SNPRINTF + /* + * Description: The snprintf_s function is equivalent to the snprintf function except for the Parameter: + * destMax/count and the explicit runtime-constraints violation + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: count - do not write more than count bytes to strDest(not including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning -1 when truncation occurs + */ + SECUREC_API int snprintf_s(char *strDest, size_t destMax, size_t count, const char *format, + ...) SECUREC_ATTRIBUTE(4, 5); +#endif + +#if SECUREC_SNPRINTF_TRUNCATED + /* + * Description: The vsnprintf_truncated_s function is equivalent to the vsnprintf_s function except + * no count Parameter: and Return: value + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning destMax - 1 when truncation occurs + */ + SECUREC_API int vsnprintf_truncated_s(char *strDest, size_t destMax, const char *format, + va_list argList) SECUREC_ATTRIBUTE(3, 0); + + /* + * Description: The snprintf_truncated_s function is equivalent to the snprintf_2 function except + * no count Parameter: and Return: value + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null byte ('\0')) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null byte ('\0')), + * If an error occurred Return: -1.Pay special attention to returning destMax - 1 when truncation occurs + */ + SECUREC_API int snprintf_truncated_s(char *strDest, size_t destMax, + const char *format, ...) SECUREC_ATTRIBUTE(3, 4); +#endif + +#if SECUREC_ENABLE_SCANF + /* + * Description: The scanf_s function is equivalent to fscanf_s with the argument stdin + * interposed before the arguments to scanf_s + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int scanf_s(const char *format, ...); +#endif + +#if SECUREC_ENABLE_VSCANF + /* + * Description: The vscanf_s function is equivalent to scanf_s, with the variable argument list replaced by argList + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vscanf_s(const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SSCANF + /* + * Description: The sscanf_s function is equivalent to fscanf_s, except that input is obtained from a + * string (specified by the argument buffer) rather than from a stream + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int sscanf_s(const char *buffer, const char *format, ...); +#endif + +#if SECUREC_ENABLE_VSSCANF + /* + * Description: The vsscanf_s function is equivalent to sscanf_s, with the variable argument list + * replaced by argList + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vsscanf_s(const char *buffer, const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_FSCANF + /* + * Description: The fscanf_s function is equivalent to fscanf except that the c, s, and [ conversion specifiers + * apply to a pair of arguments (unless assignment suppression is indicated by a*) + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int fscanf_s(FILE *stream, const char *format, ...); +#endif + +#if SECUREC_ENABLE_VFSCANF + /* + * Description: The vfscanf_s function is equivalent to fscanf_s, with the variable argument list + * replaced by argList + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vfscanf_s(FILE *stream, const char *format, va_list argList); +#endif + +#if SECUREC_ENABLE_STRTOK + /* + * Description: The strtok_s function parses a string into a sequence of strToken, + * replace all characters in strToken string that match to strDelimit set with 0. + * On the first call to strtok_s the string to be parsed should be specified in strToken. + * In each subsequent call that should parse the same string, strToken should be NULL + * Parameter: strToken - the string to be delimited + * Parameter: strDelimit -specifies a set of characters that delimit the tokens in the parsed string + * Parameter: context -is a pointer to a char * variable that is used internally by strtok_s function + * Return: On the first call returns the address of the first non \0 character, otherwise NULL is returned. + * In subsequent calls, the strtoken is set to NULL, and the context set is the same as the previous call, + * return NULL if the *context string length is equal 0, otherwise return *context. + */ + SECUREC_API char *strtok_s(char *strToken, const char *strDelimit, char **context); +#endif + +#if SECUREC_ENABLE_GETS && SECUREC_IN_KERNEL == 0 + /* + * Description: The gets_s function reads at most one less than the number of characters specified + * by destMax from the stream pointed to by stdin, into the array pointed to by buffer + * Parameter: buffer - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating null character) + * Return: buffer if there was no runtime-constraint violation,If an error occurred Return: NULL. + */ + SECUREC_API char *gets_s(char *buffer, size_t destMax); +#endif + + +#if SECUREC_ENABLE_WCHAR_FUNC +#if SECUREC_ENABLE_MEMCPY + /* + * Description: The wmemcpy_s function copies n successive wide characters from the object pointed to + * by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wmemcpy_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count); +#endif + +#if SECUREC_ENABLE_MEMMOVE + /* + * Description: The wmemmove_s function copies n successive wide characters from the object + * pointed to by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: src -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wmemmove_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count); +#endif + +#if SECUREC_ENABLE_STRCPY + /* + * Description: The wcscpy_s function copies the wide string pointed to by strSrc (including theterminating + * null wide character) into the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcscpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCPY + /* + * Description: The wcsncpy_s function copies not more than n successive wide characters (not including the + * terminating null wide character) from the array pointed to by strSrc to the array pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcsncpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRCAT + /* + * Description: The wcscat_s function appends a copy of the wide string pointed to by strSrc (including the + * terminating null wide character) to the end of the wide string pointed to by strDest + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcscat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc); +#endif + +#if SECUREC_ENABLE_STRNCAT + /* + * Description: The wcsncat_s function appends not more than n successive wide characters (not including the + * terminating null wide character) from the array pointed to by strSrc to the end of the wide string pointed to + * by strDest. + * Parameter: strDest - destination address + * Parameter: destMax -The maximum length of destination buffer(including the terminating wide character) + * Parameter: strSrc -source address + * Parameter: count -copies count wide characters from the src + * Return: EOK if there was no runtime-constraint violation + */ + SECUREC_API errno_t wcsncat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count); +#endif + +#if SECUREC_ENABLE_STRTOK + /* + * Description: The wcstok_s function is the wide-character equivalent of the strtok_s function + * Parameter: strToken - the string to be delimited + * Parameter: strDelimit -specifies a set of characters that delimit the tokens in the parsed string + * Parameter: context -is a pointer to a char * variable that is used internally by strtok_s function + * Return: a pointer to the first character of a token, or a null pointer if there is no token + * or there is a runtime-constraint violation. + */ + SECUREC_API wchar_t *wcstok_s(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context); +#endif + +#if SECUREC_ENABLE_VSPRINTF + /* + * Description: The vswprintf_s function is the wide-character equivalent of the vsprintf_s function + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null ) + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of characters printed(not including the terminating null wide characte), + * If an error occurred Return: -1. + */ + SECUREC_API int vswprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SPRINTF + + /* + * Description: The swprintf_s function is the wide-character equivalent of the sprintf_s function + * Parameter: strDest - produce output according to a format ,write to the character string strDest + * Parameter: destMax - The maximum length of destination buffer(including the terminating null ) + * Parameter: format - fromat string + * Return: the number of characters printed(not including the terminating null wide characte), + * If an error occurred Return: -1. + */ + SECUREC_API int swprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_FSCANF + /* + * Description: The fwscanf_s function is the wide-character equivalent of the fscanf_s function + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int fwscanf_s(FILE *stream, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VFSCANF + /* + * Description: The vfwscanf_s function is the wide-character equivalent of the vfscanf_s function + * Parameter: stream - stdio file stream + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vfwscanf_s(FILE *stream, const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SCANF + /* + * Description: The wscanf_s function is the wide-character equivalent of the scanf_s function + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int wscanf_s(const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VSCANF + /* + * Description: The vwscanf_s function is the wide-character equivalent of the vscanf_s function + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vwscanf_s(const wchar_t *format, va_list argList); +#endif + +#if SECUREC_ENABLE_SSCANF + /* + * Description: The swscanf_s function is the wide-character equivalent of the sscanf_s function + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int swscanf_s(const wchar_t *buffer, const wchar_t *format, ...); +#endif + +#if SECUREC_ENABLE_VSSCANF + /* + * Description: The vswscanf_s function is the wide-character equivalent of the vsscanf_s function + * Parameter: buffer - read character from buffer + * Parameter: format - fromat string + * Parameter: argList - instead of a variable number of arguments + * Return: the number of input items assigned, If an error occurred Return: -1. + */ + SECUREC_API int vswscanf_s(const wchar_t *buffer, const wchar_t *format, va_list argList); +#endif +#endif /* SECUREC_ENABLE_WCHAR_FUNC */ +#endif + + /* those functions are used by macro ,must declare hare , also for without function declaration warning */ + extern errno_t strncpy_error(char *strDest, size_t destMax, const char *strSrc, size_t count); + extern errno_t strcpy_error(char *strDest, size_t destMax, const char *strSrc); + +#if SECUREC_WITH_PERFORMANCE_ADDONS + /* those functions are used by macro */ + extern errno_t memset_sOptAsm(void *dest, size_t destMax, int c, size_t count); + extern errno_t memset_sOptTc(void *dest, size_t destMax, int c, size_t count); + extern errno_t memcpy_sOptAsm(void *dest, size_t destMax, const void *src, size_t count); + extern errno_t memcpy_sOptTc(void *dest, size_t destMax, const void *src, size_t count); + +/* strcpy_sp is a macro, NOT a function in performance optimization mode. */ +#define strcpy_sp(dest, destMax, src) ((__builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRCPY_SM((dest), (destMax), (src)) : \ + strcpy_s((dest), (destMax), (src))) + +/* strncpy_sp is a macro, NOT a function in performance optimization mode. */ +#define strncpy_sp(dest, destMax, src, count) ((__builtin_constant_p((count)) && \ + __builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRNCPY_SM((dest), (destMax), (src), (count)) : \ + strncpy_s((dest), (destMax), (src), (count))) + +/* strcat_sp is a macro, NOT a function in performance optimization mode. */ +#define strcat_sp(dest, destMax, src) ((__builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRCAT_SM((dest), (destMax), (src)) : \ + strcat_s((dest), (destMax), (src))) + +/* strncat_sp is a macro, NOT a function in performance optimization mode. */ +#define strncat_sp(dest, destMax, src, count) ((__builtin_constant_p((count)) && \ + __builtin_constant_p((destMax)) && \ + __builtin_constant_p((src))) ? \ + SECUREC_STRNCAT_SM((dest), (destMax), (src), (count)) : \ + strncat_s((dest), (destMax), (src), (count))) + +/* memcpy_sp is a macro, NOT a function in performance optimization mode. */ +#define memcpy_sp(dest, destMax, src, count) (__builtin_constant_p((count)) ? \ + (SECUREC_MEMCPY_SM((dest), (destMax), (src), (count))) : \ + (__builtin_constant_p((destMax)) ? \ + (((size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & \ + (unsigned long long)(-2)) < SECUREC_MEM_MAX_LEN)) ? \ + memcpy_sOptTc((dest), (destMax), (src), (count)) : ERANGE) : \ + memcpy_sOptAsm((dest), (destMax), (src), (count)))) + +/* memset_sp is a macro, NOT a function in performance optimization mode. */ +#define memset_sp(dest, destMax, c, count) (__builtin_constant_p((count)) ? \ + (SECUREC_MEMSET_SM((dest), (destMax), (c), (count))) : \ + (__builtin_constant_p((destMax)) ? \ + (((size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & \ + (unsigned long long)(-2)) < SECUREC_MEM_MAX_LEN)) ? \ + memset_sOptTc((dest), (destMax), (c), (count)) : ERANGE) : \ + memset_sOptAsm((dest), (destMax), (c), (count)))) +#else +#define strcpy_sp strcpy_s +#define strncpy_sp strncpy_s +#define strcat_sp strcat_s +#define strncat_sp strncat_s +#define memcpy_sp memcpy_s +#define memset_sp memset_s +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* __SECUREC_H__5D13A042_DC3F_4ED9_A8D1_882811274C27 */ + diff --git a/third_party/securec/include/securectype.h b/third_party/securec/include/securectype.h new file mode 100644 index 00000000..7d7dc190 --- /dev/null +++ b/third_party/securec/include/securectype.h @@ -0,0 +1,546 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* [Standardize-exceptions]: Performance-sensitive + * [reason]: Strict parameter verification has been done before use + */ + +#ifndef __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 +#define __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 + +#ifndef SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#if defined(__STDC_WANT_SECURE_LIB__) && __STDC_WANT_SECURE_LIB__ == 0 +/* Security functions have been provided since vs2005, default use of system library functions */ +#define SECUREC_USING_STD_SECURE_LIB 0 +#else +#define SECUREC_USING_STD_SECURE_LIB 1 +#endif +#else +#define SECUREC_USING_STD_SECURE_LIB 0 +#endif +#endif + + +/* Compatibility with older Secure C versions, shielding VC symbol redefinition warning */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 && SECUREC_USING_STD_SECURE_LIB == 0 +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 1 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 1 +#endif +#else /* MSC VER */ +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 0 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 0 +#endif +#endif + +#if SECUREC_DISABLE_CRT_FUNC +#ifdef __STDC_WANT_SECURE_LIB__ +#undef __STDC_WANT_SECURE_LIB__ +#endif +#define __STDC_WANT_SECURE_LIB__ 0 +#endif + +#if SECUREC_DISABLE_CRT_IMP +#ifdef _CRTIMP_ALTERNATIVE +#undef _CRTIMP_ALTERNATIVE +#endif +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif + +/* Compile in kernel under macro control */ +#ifndef SECUREC_IN_KERNEL +#ifdef __KERNEL__ +#define SECUREC_IN_KERNEL 1 +#else +#define SECUREC_IN_KERNEL 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#endif +#else /* SECUREC_IN_KERNEL */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 1 +#endif +#endif + + +/* Default secure function declaration, default declarations for non-standard functions */ +#ifndef SECUREC_SNPRINTF_TRUNCATED +#define SECUREC_SNPRINTF_TRUNCATED 1 +#endif + +#if SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +/* Declare secure functions that are not available in the vs compiler */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +/* vs 2005 have vsnprintf_s function */ +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +/* vs 2005 have vsnprintf_s function Adapt the snprintf_s of the security function */ +#define snprintf_s _snprintf_s +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +/* befor vs 2010 do not have v functions */ +#if _MSC_VER <= 1600 || defined(SECUREC_FOR_V_SCANFS) +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#endif + +#else /* _MSC_VER */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 0 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 0 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 0 +#endif +#ifndef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 0 +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 0 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 0 +#endif + +#else /* SECUREC_USE_STD_SECURE_LIB */ + +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 1 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 1 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 1 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#ifndef SECUREC_ENABLE_SCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF 1 +#else +#define SECUREC_ENABLE_SCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VSCANF 1 +#else +#define SECUREC_ENABLE_VSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_FSCANF 1 +#else +#define SECUREC_ENABLE_FSCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VFSCANF 1 +#else +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 1 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 1 +#endif +#endif /* SECUREC_USE_STD_SECURE_LIB */ + +#if SECUREC_ENABLE_SCANF_FILE == 0 +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#if SECUREC_ENABLE_VFSCANF +#undef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#if SECUREC_ENABLE_SCANF +#undef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif + +#endif + +#if SECUREC_IN_KERNEL +#include +#include +#else +#include +#include +#include +#endif + +/* If you need high performance, enable the SECUREC_WITH_PERFORMANCE_ADDONS macro, default is enable . + * The macro is automatically closed on the windows platform and linux kernel + */ +#ifndef SECUREC_WITH_PERFORMANCE_ADDONS +#if SECUREC_IN_KERNEL +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#else +#define SECUREC_WITH_PERFORMANCE_ADDONS 1 +#endif +#endif + +/* if enable SECUREC_COMPATIBLE_WIN_FORMAT, the output format will be compatible to Windows. */ +#if (defined(_WIN32) || defined(_WIN64) || defined(_MSC_VER)) && !defined(SECUREC_COMPATIBLE_LINUX_FORMAT) +#if !defined(SECUREC_COMPATIBLE_WIN_FORMAT) +#define SECUREC_COMPATIBLE_WIN_FORMAT +#endif +#endif + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) +/* in windows platform, can't use optimized function for there is no __builtin_constant_p like function */ +/* If need optimized macro, can define this: define __builtin_constant_p(x) 0 */ +#ifdef SECUREC_WITH_PERFORMANCE_ADDONS +#undef SECUREC_WITH_PERFORMANCE_ADDONS +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#endif +#endif + +#if defined(__VXWORKS__) || defined(__vxworks) || defined(__VXWORKS) || defined(_VXWORKS_PLATFORM_) || \ + defined(SECUREC_VXWORKS_VERSION_5_4) +#if !defined(SECUREC_VXWORKS_PLATFORM) +#define SECUREC_VXWORKS_PLATFORM +#endif +#endif + +/* if enable SECUREC_COMPATIBLE_LINUX_FORMAT, the output format will be compatible to Linux. */ +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) +#if !defined(SECUREC_COMPATIBLE_LINUX_FORMAT) +#define SECUREC_COMPATIBLE_LINUX_FORMAT +#endif +#endif + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +#include +#endif + +/* add the -DSECUREC_SUPPORT_FORMAT_WARNING compiler option to supoort -Wformat. + * default does not check the format is that the same data type in the actual code + * in the product is different in the original data type definition of VxWorks and Linux. + */ +#ifndef SECUREC_SUPPORT_FORMAT_WARNING +#define SECUREC_SUPPORT_FORMAT_WARNING 0 +#endif + +/* SECUREC_PCLINT for tool do not recognize __attribute__ just for pclint */ +#if SECUREC_SUPPORT_FORMAT_WARNING && !defined(SECUREC_PCLINT) +#define SECUREC_ATTRIBUTE(x, y) __attribute__((format(printf, (x), (y)))) +#else +#define SECUREC_ATTRIBUTE(x, y) +#endif + +/* SECUREC_PCLINT for tool do not recognize __builtin_expect, just for pclint */ +#if defined(__GNUC__) && \ + ((__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 3))) && \ + !defined(SECUREC_PCLINT) +/* This is a built-in function that can be used without a declaration, if you encounter an undeclared compilation alarm, + * you can add -DSECUREC_NEED_BUILTIN_EXPECT_DECLARE to complier options + */ +#if defined(SECUREC_NEED_BUILTIN_EXPECT_DECLARE) +long __builtin_expect(long exp, long c); +#endif +#define SECUREC_LIKELY(x) __builtin_expect(!!(x), 1) +#define SECUREC_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define SECUREC_LIKELY(x) (x) +#define SECUREC_UNLIKELY(x) (x) +#endif + +/* define the max length of the string */ +#ifndef SECUREC_STRING_MAX_LEN +#define SECUREC_STRING_MAX_LEN (0x7fffffffUL) +#endif +#define SECUREC_WCHAR_STRING_MAX_LEN (SECUREC_STRING_MAX_LEN / sizeof(wchar_t)) + +/* add SECUREC_MEM_MAX_LEN for memcpy and memmove */ +#ifndef SECUREC_MEM_MAX_LEN +#define SECUREC_MEM_MAX_LEN (0x7fffffffUL) +#endif +#define SECUREC_WCHAR_MEM_MAX_LEN (SECUREC_MEM_MAX_LEN / sizeof(wchar_t)) + +#if SECUREC_STRING_MAX_LEN > 0x7fffffff +#error "max string is 2G" +#endif + +#if (defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if (__SIZEOF_POINTER__ != 4) && (__SIZEOF_POINTER__ != 8) +#error "unsupported system" +#endif +#endif + +#if defined(_WIN64) || defined(WIN64) || defined(__LP64__) || defined(_LP64) +#define SECUREC_ON_64BITS +#endif + +#if (!defined(SECUREC_ON_64BITS) && defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if __SIZEOF_POINTER__ == 8 +#define SECUREC_ON_64BITS +#endif +#endif + +#if defined(__SVR4) || defined(__svr4__) +#define SECUREC_ON_SOLARIS +#endif + +#if (defined(__hpux) || defined(_AIX) || defined(SECUREC_ON_SOLARIS)) +#define SECUREC_ON_UNIX +#endif + +/* codes should run under the macro SECUREC_COMPATIBLE_LINUX_FORMAT in unknow system on default, + * and strtold. The function + * strtold is referenced first at ISO9899:1999(C99), and some old compilers can + * not support these functions. Here provides a macro to open these functions: + * SECUREC_SUPPORT_STRTOLD -- if defined, strtold will be used + */ +#ifndef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) +#if defined(__USE_ISOC99) || \ + (defined(_AIX) && defined(_ISOC99_SOURCE)) || \ + (defined(__hpux) && defined(__ia64)) || \ + (defined(SECUREC_ON_SOLARIS) && (!defined(_STRICT_STDC) && !defined(__XOPEN_OR_POSIX)) || \ + defined(_STDC_C99) || defined(__EXTENSIONS__)) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 1 +#endif +#endif +#if ((defined(SECUREC_WRLINUX_BELOW4) || defined(_WRLINUX_BELOW4_))) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#endif +#endif + + +#if SECUREC_WITH_PERFORMANCE_ADDONS + +#ifndef SECUREC_TWO_MIN +#define SECUREC_TWO_MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +/* for strncpy_s performance optimization */ +#define SECUREC_STRNCPY_SM(dest, destMax, src, count) \ + (((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (SECUREC_TWO_MIN((size_t)(count), strlen(src)) + 1) <= (size_t)(destMax)) ? \ + (((size_t)(count) < strlen(src)) ? (memcpy((dest), (src), (count)), *((char *)(dest) + (count)) = '\0', EOK) : \ + (memcpy((dest), (src), strlen(src) + 1), EOK)) : (strncpy_error((dest), (destMax), (src), (count)))) + +#define SECUREC_STRCPY_SM(dest, destMax, src) \ + (((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (strlen(src) + 1) <= (size_t)(destMax)) ? (memcpy((dest), (src), strlen(src) + 1), EOK) : \ + (strcpy_error((dest), (destMax), (src)))) + +/* for strcat_s performance optimization */ +#if defined(__GNUC__) +#define SECUREC_STRCAT_SM(dest, destMax, src) ({ \ + int catRet = EOK; \ + if ((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN)) { \ + char *catTmpDst = (char *)(dest); \ + size_t catRestSize = (destMax); \ + while (catRestSize > 0 && *catTmpDst != '\0') { \ + ++catTmpDst; \ + --catRestSize; \ + } \ + if (catRestSize == 0) { \ + catRet = EINVAL; \ + } else if ((strlen(src) + 1) <= catRestSize) { \ + memcpy(catTmpDst, (src), strlen(src) + 1); \ + catRet = EOK; \ + } else { \ + catRet = ERANGE; \ + } \ + if (catRet != EOK) { \ + catRet = strcat_s((dest), (destMax), (src)); \ + } \ + } else { \ + catRet = strcat_s((dest), (destMax), (src)); \ + } \ + catRet; \ +}) +#else +#define SECUREC_STRCAT_SM(dest, destMax, src) strcat_s((dest), (destMax), (src)) +#endif + +/* for strncat_s performance optimization */ +#if defined(__GNUC__) +#define SECUREC_STRNCAT_SM(dest, destMax, src, count) ({ \ + int ncatRet = EOK; \ + if ((void *)(dest) != NULL && (void *)(src) != NULL && (size_t)(destMax) > 0 && \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN) && \ + (((unsigned long long)(count) & (unsigned long long)(-2)) < SECUREC_STRING_MAX_LEN)) { \ + char *ncatTmpDest = (char *)(dest); \ + size_t ncatRestSize = (size_t)(destMax); \ + while (ncatRestSize > 0 && *ncatTmpDest != '\0') { \ + ++ncatTmpDest; \ + --ncatRestSize; \ + } \ + if (ncatRestSize == 0) { \ + ncatRet = EINVAL; \ + } else if ((SECUREC_TWO_MIN((count), strlen(src)) + 1) <= ncatRestSize) { \ + if ((size_t)(count) < strlen(src)) { \ + memcpy(ncatTmpDest, (src), (count)); \ + *(ncatTmpDest + (count)) = '\0'; \ + } else { \ + memcpy(ncatTmpDest, (src), strlen(src) + 1); \ + } \ + } else { \ + ncatRet = ERANGE; \ + } \ + if (ncatRet != EOK) { \ + ncatRet = strncat_s((dest), (destMax), (src), (count)); \ + } \ + } else { \ + ncatRet = strncat_s((dest), (destMax), (src), (count)); \ + } \ + ncatRet; \ +}) +#else +#define SECUREC_STRNCAT_SM(dest, destMax, src, count) strncat_s((dest), (destMax), (src), (count)) +#endif + +/* SECUREC_MEMCPY_SM do NOT check buffer overlap by default */ +#define SECUREC_MEMCPY_SM(dest, destMax, src, count) \ + (!(((size_t)(destMax) == 0) || \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((size_t)(count) > (size_t)(destMax)) || ((void *)(dest)) == NULL || ((void *)(src) == NULL))? \ + (memcpy((dest), (src), (count)), EOK) : \ + (memcpy_s((dest), (destMax), (src), (count)))) + +#define SECUREC_MEMSET_SM(dest, destMax, c, count) \ + (!(((size_t)(destMax) == 0) || \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((void *)(dest) == NULL) || ((size_t)(count) > (size_t)(destMax))) ? \ + (memset((dest), (c), (count)), EOK) : \ + (memset_s((dest), (destMax), (c), (count)))) + +#endif +#endif /* __SECURECTYPE_H__A7BBB686_AADA_451B_B9F9_44DACDAE18A7 */ + diff --git a/third_party/securec/src/CMakeLists.txt b/third_party/securec/src/CMakeLists.txt new file mode 100644 index 00000000..28f30c51 --- /dev/null +++ b/third_party/securec/src/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(. SECUREC_SRCS) + +add_library(securec ${SECUREC_SRCS}) diff --git a/third_party/securec/src/fscanf_s.c b/third_party/securec/src/fscanf_s.c new file mode 100644 index 00000000..9d1a2874 --- /dev/null +++ b/third_party/securec/src/fscanf_s.c @@ -0,0 +1,55 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The fscanf_s function is equivalent to fscanf except that the c, s, + * and [ conversion specifiers apply to a pair of arguments (unless assignment suppression is indicated by a*) + * The fscanf function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The convered value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int fscanf_s(FILE *stream, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vfscanf_s(stream, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/fwscanf_s.c b/third_party/securec/src/fwscanf_s.c new file mode 100644 index 00000000..454ac242 --- /dev/null +++ b/third_party/securec/src/fwscanf_s.c @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The fwscanf_s function is the wide-character equivalent of the fscanf_s function + * The fwscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int fwscanf_s(FILE *stream, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vfwscanf_s(stream, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/gets_s.c b/third_party/securec/src/gets_s.c new file mode 100644 index 00000000..e0274aa6 --- /dev/null +++ b/third_party/securec/src/gets_s.c @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securecutil.h" + +static void SecTrimCRLF(char *buffer, size_t len) +{ + int i; + /* No need to determine whether integer overflow exists */ + for (i = (int)(len - 1); i >= 0 && (buffer[i] == '\r' || buffer[i] == '\n'); --i) { + buffer[i] = '\0'; + } + return; +} + +/* + * + * The gets_s function reads at most one less than the number of characters + * specified by destMax from the stream pointed to by stdin, into the array pointed to by buffer + * The line consists of all characters up to and including + * the first newline character ('\n'). gets_s then replaces the newline + * character with a null character ('\0') before returning the line. + * If the first character read is the end-of-file character, a null character + * is stored at the beginning of buffer and NULL is returned. + * + * + * buffer Storage location for input string. + * numberOfElements The size of the buffer. + * + * + * buffer is updated + * + * + * buffer Successful operation + * NULL Improper parameter or read fail + */ +char *gets_s(char *buffer, size_t numberOfElements) +{ + size_t len; +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + size_t bufferSize = ((numberOfElements == (size_t)-1) ? SECUREC_STRING_MAX_LEN : numberOfElements); +#else + size_t bufferSize = numberOfElements; +#endif + + if (buffer == NULL || bufferSize == 0 || bufferSize > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("gets_s"); + return NULL; + } + + if (fgets(buffer, (int)bufferSize, stdin) == NULL) { + return NULL; + } + + len = strlen(buffer); + if (len > 0 && len < bufferSize) { + SecTrimCRLF(buffer, len); + } + + return buffer; +} + diff --git a/third_party/securec/src/input.inl b/third_party/securec/src/input.inl new file mode 100644 index 00000000..6691f8a2 --- /dev/null +++ b/third_party/securec/src/input.inl @@ -0,0 +1,2124 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INPUT_INL_5D13A042_DC3F_4ED9_A8D1_882811274C27 +#define INPUT_INL_5D13A042_DC3F_4ED9_A8D1_882811274C27 + +#if SECUREC_IN_KERNEL +#include +#ifndef EOF +#define EOF (-1) +#endif +#else +#if !defined(SECUREC_SYSAPI4VXWORKS) && !defined(SECUREC_CTYPE_MACRO_ADAPT) +#include +#ifdef SECUREC_FOR_WCHAR +#include /* for iswspace */ +#endif +#endif +#endif + +#define SECUREC_NUM_WIDTH_SHORT 0 +#define SECUREC_NUM_WIDTH_INT 1 +#define SECUREC_NUM_WIDTH_LONG 2 +#define SECUREC_NUM_WIDTH_LONG_LONG 3 /* also long double */ + +#define SECUREC_BUF_EXT_MUL 2 +#define SECUREC_BUFFERED_BLOK_SIZE 1024 + +#if defined(SECUREC_VXWORKS_PLATFORM) && !defined(va_copy) && !defined(__va_copy) +/* the name is the same as system macro. */ +#define __va_copy(d, s) do { \ + size_t size_of_d = (size_t)sizeof(d); \ + size_t size_of_s = (size_t)sizeof(s); \ + if (size_of_d != size_of_s) { \ + (void)memcpy((d), (s), sizeof(va_list)); \ + } else { \ + (void)memcpy(&(d), &(s), sizeof(va_list)); \ + } \ +} SECUREC_WHILE_ZERO +#endif + + +#define SECUREC_MULTI_BYTE_MAX_LEN 6 +/* Record a flag for each bit */ +#define SECUREC_BRACKET_INDEX(x) ((unsigned int)(x) >> 3) +#define SECUREC_BRACKET_VALUE(x) ((unsigned char)(1 << ((unsigned int)(x) & 7))) + + +/* Compatibility macro name cannot be modifie */ +#ifndef UNALIGNED +#if !(defined(_M_IA64)) && !(defined(_M_AMD64)) +#define UNALIGNED +#else +#define UNALIGNED __unaligned +#endif +#endif + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* Max 64bit value is 0xffffffffffffffff */ +#define SECUREC_MAX_64BITS_VALUE 18446744073709551615ULL +#define SECUREC_MAX_64BITS_VALUE_DIV_TEN 1844674407370955161ULL +#define SECUREC_MAX_64BITS_VALUE_CUT_LAST_DIGIT 18446744073709551610ULL +#define SECUREC_MIN_64BITS_NEG_VALUE 9223372036854775808ULL +#define SECUREC_MAX_64BITS_POS_VALUE 9223372036854775807ULL +#define SECUREC_MIN_32BITS_NEG_VALUE 2147483648ULL +#define SECUREC_MAX_32BITS_POS_VALUE 2147483647ULL +#define SECUREC_MAX_32BITS_VALUE 4294967295ULL +#define SECUREC_MAX_32BITS_VALUE_INC 4294967296ULL +#define SECUREC_MAX_32BITS_VALUE_DIV_TEN 429496729ULL +#define SECUREC_LONG_BIT_NUM ((unsigned int)(sizeof(long) << 3U)) + +#define SECUREC_LONG_HEX_BEYOND_MAX(number) (((number) >> (SECUREC_LONG_BIT_NUM - 4U)) > 0) +#define SECUREC_LONG_OCTAL_BEYOND_MAX(number) (((number) >> (SECUREC_LONG_BIT_NUM - 3U)) > 0) + +#define SECUREC_QWORD_HEX_BEYOND_MAX(number) (((number) >> (64U - 4U)) > 0) +#define SECUREC_QWORD_OCTAL_BEYOND_MAX(number) (((number) >> (64U - 3U)) > 0) + +#define SECUREC_LP64_BIT_WIDTH 64 +#define SECUREC_LP32_BIT_WIDTH 32 + +#endif + +#define SECUREC_CHAR(x) (x) +#define SECUREC_BRACE '{' /* [ to { */ + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_SCANF_BRACKET_CONDITION(comChr, ch, table, mask) ((comChr) == SECUREC_BRACE && \ + (table) != NULL && \ + (((table)[((unsigned int)(int)(ch) & SECUREC_CHAR_MASK) >> 3] ^ (mask)) & \ + (1 << ((unsigned int)(int)(ch) & 7)))) +#else +#define SECUREC_SCANF_BRACKET_CONDITION(comChr, ch, table, mask) ((comChr) == SECUREC_BRACE && \ + (((table)[((unsigned char)(ch) & 0xff) >> 3] ^ (mask)) & (1 << ((unsigned char)(ch) & 7)))) +#endif +#define SECUREC_SCANF_STRING_CONDITION(comChr, ch) ((comChr) == SECUREC_CHAR('s') && \ + (!((ch) >= SECUREC_CHAR('\t') && (ch) <= SECUREC_CHAR('\r')) && (ch) != SECUREC_CHAR(' '))) + +/* Do not use |= optimize this code, it will cause compiling warning */ +/* only supports wide characters with a maximum length of two bytes */ +#define SECUREC_BRACKET_SET_BIT(table, ch) do { \ + unsigned int tableIndex = SECUREC_BRACKET_INDEX(((unsigned int)(int)(ch) & SECUREC_CHAR_MASK)); \ + unsigned int tableValue = SECUREC_BRACKET_VALUE(((unsigned int)(int)(ch) & SECUREC_CHAR_MASK)); \ + (table)[tableIndex] = (unsigned char)((table)[tableIndex] | tableValue); \ +} SECUREC_WHILE_ZERO + +#ifdef SECUREC_FOR_WCHAR +/* table size is 32 x 256 */ +#define SECUREC_BRACKET_TABLE_SIZE 8192 +#define SECUREC_EOF WEOF +#define SECUREC_MB_LEN 16 /* max. # bytes in multibyte char ,see MB_LEN_MAX */ +/* int to unsigned int clear e571 */ +#define SECUREC_IS_DIGIT(chr) (!((unsigned int)(int)(chr) & 0xff00) && isdigit(((unsigned int)(int)(chr) & 0x00ff))) +#define SECUREC_IS_XDIGIT(chr) (!((unsigned int)(int)(chr) & 0xff00) && isxdigit(((unsigned int)(int)(chr) & 0x00ff))) +#define SECUREC_IS_SPACE(chr) iswspace((wint_t)(int)(chr)) +#else +#define SECUREC_BRACKET_TABLE_SIZE 32 +#define SECUREC_EOF EOF +#define SECUREC_IS_DIGIT(chr) isdigit((unsigned char)(chr) & 0x00ff) +#define SECUREC_IS_XDIGIT(chr) isxdigit((unsigned char)(chr) & 0x00ff) +#define SECUREC_IS_SPACE(chr) isspace((unsigned char)(chr) & 0x00ff) +#endif + + +static SecInt SecSkipSpaceChar(SecFileStream *stream, int *counter); +static SecInt SecGetChar(SecFileStream *stream, int *counter); +static void SecUnGetChar(SecInt ch, SecFileStream *stream, int *counter); + +typedef struct { +#ifdef SECUREC_FOR_WCHAR + unsigned char *table; /* default NULL */ +#else + unsigned char table[SECUREC_BRACKET_TABLE_SIZE]; /* Array length is large enough in application scenarios */ +#endif + unsigned char mask; /* default 0 */ +} SecBracketTable; + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_INIT_BRACKET_TABLE { NULL, 0 } +#else +#define SECUREC_INIT_BRACKET_TABLE { { 0 }, 0 } +#endif + +#if SECUREC_ENABLE_SCANF_FLOAT +typedef struct { + size_t floatStrSize; /* tialization must be length of buffer in charater */ + size_t floatStrUsedLen; /* store float string len */ + SecChar buffer[SECUREC_FLOAT_BUFSIZE + 1]; + SecChar *floatStr; /* Initialization must point to buffer */ + SecChar *allocatedFloatStr; /* Initialization must be NULL to store alloced point */ +} SecFloatSpec; +#endif + +typedef struct { + SecUnsignedInt64 number64; + unsigned long number; + int numberWidth; /* 0 = SHORT, 1 = int, > 1 long or L_DOUBLE */ + int isInt64Arg; /* 1 for 64-bit integer, 0 otherwise */ + int negative; /* 0 is positive */ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + int beyondMax; /* Non-zero means beyond */ +#endif + void *argPtr; /* Variable parameter pointer */ + size_t arrayWidth; /* length of pointer Variable parameter, in charaters */ + int width; /* width number in format */ + int widthSet; /* 0 is not set width in format */ + int comChr; /* Lowercase format conversion characters */ + int oriComChr; /* store number conversion */ + signed char isWChar; /* -1/0 not wchar, 1 for wchar */ + char suppress; /* 0 is not have %* in format */ +} SecScanSpec; + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +#define SECUREC_INIT_NUMBER_SPEC { 0, 0, 0, 0, 0, 0, NULL, 0, 0, 0, 0, 0, 0 } +#else +#define SECUREC_INIT_NUMBER_SPEC { 0, 0, 0, 0, 0, 0, NULL, 0, 0, 0, 0, 0 } +#endif + +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_GETC fgetwc +#define SECUREC_UN_GETC ungetwc +#define SECUREC_CHAR_MASK 0xffff +#else +#define SECUREC_GETC fgetc +#define SECUREC_UN_GETC ungetc +#define SECUREC_CHAR_MASK 0xff +#endif + +/* + * Determine if it is a 64-bit pointer function + * return 0 is not ,1 is 64bit pointer + */ +static int SecIs64BitPtr(size_t sizeOfVoidStar) +{ + /* point size is 4 or 8 , Under the 64 bit system, the value not 0 */ + /* to clear e778 */ + if ((sizeOfVoidStar & sizeof(SecInt64)) != 0) { + return 1; + } + return 0; +} + +#if SECUREC_ENABLE_SCANF_FLOAT + +/* + * Convert a floating point string to a floating point number + */ +static void SecAssignFloat(const char *floatStr, int numberWidth, void *argPtr) +{ + char *endPtr = NULL; + double d; +#if SECUREC_SUPPORT_STRTOLD + if (numberWidth == SECUREC_NUM_WIDTH_LONG_LONG) { + long double d2 = strtold(floatStr, &endPtr); + *(long double UNALIGNED *)(argPtr) = d2; + return; + } +#endif + d = strtod(floatStr, &endPtr); + if (numberWidth > SECUREC_NUM_WIDTH_INT) { + *(double UNALIGNED *)(argPtr) = (double)d; + } else { + *(float UNALIGNED *)(argPtr) = (float)d; + } +} + +#ifdef SECUREC_FOR_WCHAR +/* + * Convert a floating point wchar string to a floating point number + * Success ret 0 + */ +static int SecAssignFloatW(const SecFloatSpec *floatSpec, const SecScanSpec *spec) +{ + /* convert float string */ + size_t mbsLen; + size_t tempFloatStrLen = (size_t)(floatSpec->floatStrSize + 1) * sizeof(wchar_t); + char *tempFloatStr = (char *)SECUREC_MALLOC(tempFloatStrLen); + + if (tempFloatStr == NULL) { + return -1; + } + tempFloatStr[0] = '\0'; + SECUREC_MASK_MSVC_CRT_WARNING + mbsLen = wcstombs(tempFloatStr, floatSpec->floatStr, tempFloatStrLen - 1); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (mbsLen != (size_t)-1) { + tempFloatStr[mbsLen] = '\0'; + SecAssignFloat(tempFloatStr, spec->numberWidth, spec->argPtr); + } else { + SECUREC_FREE(tempFloatStr); + return -1; + } + SECUREC_FREE(tempFloatStr); + return 0; +} +#endif +/* + * Splice floating point string + * return 0 OK + */ +static int SecUpdateFloatString(SecChar ch, SecFloatSpec *floatSpec) +{ + floatSpec->floatStr[floatSpec->floatStrUsedLen++] = ch; /* ch must be '0' - '9' */ + if (floatSpec->floatStrUsedLen < floatSpec->floatStrSize) { + return 0; + } + if (floatSpec->allocatedFloatStr == NULL) { + /* add 1 to clear ZERO LENGTH ALLOCATIONS warning */ + size_t oriBufSize = floatSpec->floatStrSize* (SECUREC_BUF_EXT_MUL * sizeof(SecChar)) + 1; + void *tmpPointer = (void *)SECUREC_MALLOC(oriBufSize); + if (tmpPointer == NULL) { + return -1; + } + if (memcpy_s(tmpPointer, oriBufSize, floatSpec->floatStr, floatSpec->floatStrSize * sizeof(SecChar)) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + floatSpec->floatStr = (SecChar *) (tmpPointer); + floatSpec->allocatedFloatStr = (SecChar *) (tmpPointer); /* use to clear free on stack warning */ + floatSpec->floatStrSize *= SECUREC_BUF_EXT_MUL; /* this is OK, oriBufSize plus 1 just clear warning */ + return 0; + } else { + /* LSD 2014.3.6 fix, replace realloc to malloc to avoid heap injection */ + size_t oriBufSize = floatSpec->floatStrSize * sizeof(SecChar); + size_t nextSize = (oriBufSize * SECUREC_BUF_EXT_MUL) + 1; /* add 1 to clear satic check tool warning */ + /* Prevents integer overflow when calculating the wide character length. + * The maximum length of SECUREC_MAX_WIDTH_LEN is enough + */ + if (nextSize <= SECUREC_MAX_WIDTH_LEN) { + void *tmpPointer = (void *)SECUREC_MALLOC(nextSize); + if (tmpPointer == NULL) { + return -1; + } + if (memcpy_s(tmpPointer, nextSize, floatSpec->floatStr, oriBufSize) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + if (memset_s(floatSpec->floatStr, oriBufSize, 0, oriBufSize) != EOK) { + SECUREC_FREE(tmpPointer); /* This is a dead code, just to meet the coding requirements */ + return -1; + } + SECUREC_FREE(floatSpec->floatStr); + + floatSpec->floatStr = (SecChar *) (tmpPointer); + floatSpec->allocatedFloatStr = (SecChar *) (tmpPointer); /* use to clear free on stack warning */ + floatSpec->floatStrSize *= SECUREC_BUF_EXT_MUL; /* this is OK, oriBufSize plus 1 just clear warning */ + return 0; + } + } + return -1; +} +#endif + +#ifndef SECUREC_FOR_WCHAR +/* LSD only multi-bytes string need isleadbyte() function */ +static int SecIsLeadByte(SecInt ch) +{ + unsigned int c = (unsigned int)ch; +#if !(defined(_MSC_VER) || defined(_INC_WCTYPE)) + return (int)(c & 0x80); +#else + return (int)isleadbyte((int)(c & 0xff)); +#endif +} +#endif + +/* + * Parsing whether it is a wide character + */ +static void SecUpdateWcharFlagByType(SecUnsignedChar ch, SecScanSpec *spec) +{ +#if defined(SECUREC_FOR_WCHAR) && (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + signed char flagForUpperType = -1; + signed char flagForLowerType = 1; +#else + signed char flagForUpperType = 1; + signed char flagForLowerType = -1; +#endif + /* if no l or h flag */ + if (spec->isWChar == 0) { + if ((ch == SECUREC_CHAR('C')) || (ch == SECUREC_CHAR('S'))) { + spec->isWChar = flagForUpperType; + } else { + spec->isWChar = flagForLowerType; + } + } + return; +} +/* + * decode %l %ll + */ +static void SecDecodeScanQualifierL(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + if (*(fmt + 1) == SECUREC_CHAR('l')) { + spec->isInt64Arg = 1; + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + ++fmt; + } else { + spec->numberWidth = SECUREC_NUM_WIDTH_LONG; +#if defined(SECUREC_ON_64BITS) && !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + /* on window 64 system sizeof long is 32bit */ + spec->isInt64Arg = 1; +#endif + spec->isWChar = 1; + } + *format = fmt; +} + +/* + * decode %I %I43 %I64 %Id %Ii %Io ... + * set finishFlag to 1 finish Flag + */ +static void SecDecodeScanQualifierI(const SecUnsignedChar **format, SecScanSpec *spec, int *finishFlag) +{ + const SecUnsignedChar *fmt = *format; + if ((*(fmt + 1) == SECUREC_CHAR('6')) && + (*(fmt + 2) == SECUREC_CHAR('4'))) { /* offset 2 for I64 */ + spec->isInt64Arg = 1; + *format = *format + 2; /* add 2 to skip I64 point to '4' next loop will inc */ + } else if ((*(fmt + 1) == SECUREC_CHAR('3')) && + (*(fmt + 2) == SECUREC_CHAR('2'))) { /* offset 2 for I32 */ + *format = *format + 2; /* add 2 to skip I32 point to '2' next loop will inc */ + } else if ((*(fmt + 1) == SECUREC_CHAR('d')) || + (*(fmt + 1) == SECUREC_CHAR('i')) || + (*(fmt + 1) == SECUREC_CHAR('o')) || + (*(fmt + 1) == SECUREC_CHAR('x')) || + (*(fmt + 1) == SECUREC_CHAR('X'))) { + spec->isInt64Arg = SecIs64BitPtr(sizeof(void *)); + } else { + /* for %I */ + spec->isInt64Arg = SecIs64BitPtr(sizeof(void *)); + *finishFlag = 1; + } +} + +static int SecDecodeScanWidth(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + while (SECUREC_IS_DIGIT(*fmt)) { + spec->widthSet = 1; + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(spec->width)) { + return -1; + } + spec->width = (int)SECUREC_MUL_TEN((unsigned int)spec->width) + (unsigned char)(*fmt - SECUREC_CHAR('0')); + ++fmt; + } + *format = fmt; + return 0; +} + +/* + * init default flags for each format + */ +static void SecSetDefaultScanSpec(SecScanSpec *spec) +{ + spec->number64 = 0; + spec->number = 0; + spec->numberWidth = SECUREC_NUM_WIDTH_INT; /* 0 = SHORT, 1 = int, > 1 long or L_DOUBLE */ + spec->isInt64Arg = 0; /* 1 for 64-bit integer, 0 otherwise */ + spec->negative = 0; +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + spec->beyondMax = 0; +#endif + spec->argPtr = NULL; + spec->arrayWidth = 0; + spec->width = 0; + spec->widthSet = 0; + spec->comChr = 0; + spec->isWChar = 0; + spec->suppress = 0; +} + +/* + * decode qualifier %I %L %h ... + * set finishFlag to 1 finish Flag + */ +static void SecDecodeScanQualifier(const SecUnsignedChar **format, SecScanSpec *spec, int *finishFlag) +{ + switch ((int)(unsigned char)(**(format))) { + case SECUREC_CHAR('F'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('N'): + break; + case SECUREC_CHAR('h'): + --spec->numberWidth; /* h for SHORT , hh for CHAR */ + spec->isWChar = -1; + break; +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('j'): + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; /* intmax_t or uintmax_t */ + spec->isInt64Arg = 1; + break; + case SECUREC_CHAR('t'): /* fall-through */ /* FALLTHRU */ +#endif + case SECUREC_CHAR('z'): +#ifdef SECUREC_ON_64BITS + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + spec->isInt64Arg = 1; +#else + spec->numberWidth = SECUREC_NUM_WIDTH_LONG; +#endif + break; + case SECUREC_CHAR('L'): /* long double */ /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('q'): + spec->numberWidth = SECUREC_NUM_WIDTH_LONG_LONG; + spec->isInt64Arg = 1; + break; + case SECUREC_CHAR('l'): + SecDecodeScanQualifierL(format, spec); + break; + case SECUREC_CHAR('w'): + spec->isWChar = 1; + break; + case SECUREC_CHAR('*'): + spec->suppress = 1; + break; + case SECUREC_CHAR('I'): + SecDecodeScanQualifierI(format, spec, finishFlag); + break; + default: + *finishFlag = 1; + break; + } + +} +/* + * decode width and qualifier in format + */ +static int SecDecodeScanFlag(const SecUnsignedChar **format, SecScanSpec *spec) +{ + const SecUnsignedChar *fmt = *format; + int finishFlag = 0; + + do { + ++fmt; /* first skip % , next seek fmt */ + /* may %*6d , so put it inside the loop */ + if (SecDecodeScanWidth(&fmt, spec) != 0) { + return -1; + } + SecDecodeScanQualifier(&fmt, spec, &finishFlag); + } while (finishFlag == 0); + *format = fmt; + return 0; +} + + + + + +/* + * Judging whether a zeroing buffer is needed according to different formats + */ +static int SecDecodeClearFormat(const SecUnsignedChar *format, int *comChr) +{ + const SecUnsignedChar *fmt = format; + /* to lowercase */ + int ch = (unsigned char)(*fmt) | (SECUREC_CHAR('a') - SECUREC_CHAR('A')); + if (!(ch == SECUREC_CHAR('c') || ch == SECUREC_CHAR('s') || ch == SECUREC_BRACE)) { + return -1; /* first argument is not a string type */ + } + if (ch == SECUREC_BRACE) { +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + if (*fmt == SECUREC_CHAR('{')) { + return -1; + } +#endif + ++fmt; + if (*fmt == SECUREC_CHAR('^')) { + ++fmt; + } + if (*fmt == SECUREC_CHAR(']')) { + ++fmt; + } + while ((*fmt != SECUREC_CHAR('\0')) && (*fmt != SECUREC_CHAR(']'))) { + ++fmt; + } + if (*fmt == SECUREC_CHAR('\0')) { + return -1; /* trunc'd format string */ + } + } + *comChr = ch; + return 0; +} + +/* + * add L'\0' for wchar string , add '\0' for char string + */ +static void SecAddEndingZero(void *ptr, const SecScanSpec *spec) +{ + *(char *)ptr = '\0'; + (void)spec; /* clear not use */ +#if SECUREC_HAVE_WCHART + if (spec->isWChar > 0) { + *(wchar_t UNALIGNED *)ptr = L'\0'; + } +#endif +} + +#ifdef SECUREC_FOR_WCHAR +/* + * Clean up the first %s %c buffer to zero for wchar version + */ +void SecClearDestBufW(const wchar_t *buffer, const wchar_t *format, va_list argList) +#else +/* + * Clean up the first %s %c buffer to zero for char version + */ +void SecClearDestBuf(const char *buffer, const char *format, va_list argList) +#endif +{ + + va_list argListSave; /* backup for argList value, this variable don't need initialized */ + SecScanSpec spec; + int comChr = 0; + const SecUnsignedChar *fmt = (const SecUnsignedChar *)format; + if (fmt == NULL) { + return; + } + + /* find first % */ + while (*fmt != SECUREC_CHAR('\0') && *fmt != SECUREC_CHAR('%')) { + ++fmt; + } + if (*fmt == SECUREC_CHAR('\0')) { + return; + } + + SecSetDefaultScanSpec(&spec); + if (SecDecodeScanFlag(&fmt, &spec) != 0) { + return; + } + + /* update wchar flag for %S %C */ + SecUpdateWcharFlagByType(*fmt, &spec); + + if (spec.suppress != 0 || SecDecodeClearFormat(fmt, &comChr) != 0) { + return; + } + + if ((buffer != NULL) && (*buffer != SECUREC_CHAR('\0')) && (comChr != SECUREC_CHAR('s'))) { + /* when buffer not empty just clear %s. + * example call sscanf by argment of (" \n", "%s", s, sizeof(s)) + */ + return; + } + (void)memset(&argListSave, 0, sizeof(va_list)); /* to clear e530 argListSave not initialized */ +#if defined(va_copy) + va_copy(argListSave, argList); +#elif defined(__va_copy) /* for vxworks */ + __va_copy(argListSave, argList); +#else + argListSave = argList; +#endif + do { + void *argPtr = (void *)va_arg(argListSave, void *); + /* Get the next argument - size of the array in characters */ + size_t arrayWidth = ((size_t)(va_arg(argListSave, size_t))) & 0xFFFFFFFFUL; + va_end(argListSave); + /* to clear e438 last value assigned not used , the compiler will optimize this code */ + (void)argListSave; + /* There is no need to judge the upper limit */ + if (arrayWidth == 0 || argPtr == NULL) { + return; + } + + /* clear one char */ + SecAddEndingZero(argPtr, &spec); + } SECUREC_WHILE_ZERO; + return; + +} + +/* + * Assign number to output buffer + */ +static void SecAssignNumber(const SecScanSpec *spec) +{ + void *argPtr = spec->argPtr; + if (spec->isInt64Arg != 0) { +#if defined(SECUREC_VXWORKS_PLATFORM) +#if defined(SECUREC_VXWORKS_PLATFORM_COMP) + *(SecInt64 UNALIGNED *)argPtr = (SecInt64)(spec->number64); +#else + /* take number64 as unsigned number unsigned to int clear Compile warning */ + *(SecInt64 UNALIGNED *)argPtr = *(SecUnsignedInt64 *)(&(spec->number64)); +#endif +#else + /* take number64 as unsigned number */ + *(SecInt64 UNALIGNED *)argPtr = (SecInt64)(spec->number64); +#endif + return; + } + if (spec->numberWidth > SECUREC_NUM_WIDTH_INT) { + /* take number as unsigned number */ + *(long UNALIGNED *)argPtr = (long)(spec->number); + } else if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + *(int UNALIGNED *)argPtr = (int)(spec->number); + } else if (spec->numberWidth == SECUREC_NUM_WIDTH_SHORT) { + /* take number as unsigned number */ + *(short UNALIGNED *)argPtr = (short)(spec->number); + } else { /* < 0 for hh format modifier */ + /* take number as unsigned number */ + *(char UNALIGNED *)argPtr = (char)(spec->number); + } +} + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* + * Judge the long bit width + */ +static int SecIsLongBitEqual(int bitNum) +{ + return (unsigned int)bitNum == SECUREC_LONG_BIT_NUM; +} +#endif +/* + * Convert hexadecimal characters to decimal value + */ +static int SecHexValueOfChar(SecInt ch) +{ + /* use isdigt Causing tool false alarms */ + return (int)((ch >= '0' && ch <= '9') ? ((unsigned char)ch - '0') : + ((((unsigned char)ch | (unsigned char)('a' - 'A')) - ('a')) + 10)); /* Adding 10 is to hex value */ +} + + + +/* + * Parse decimal character to integer for 32bit . + */ +static void SecDecodeNumberDecimal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + unsigned long decimalEdge = SECUREC_MAX_32BITS_VALUE_DIV_TEN; +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + decimalEdge = (unsigned long)SECUREC_MAX_64BITS_VALUE_DIV_TEN; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + decimalEdge = SECUREC_MAX_32BITS_VALUE_DIV_TEN; + } +#endif + if (spec->number > decimalEdge) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_TEN(spec->number); +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number == SECUREC_MUL_TEN(decimalEdge)) { + SecUnsignedInt64 number64As = (unsigned long)SECUREC_MAX_64BITS_VALUE - spec->number; + if (number64As < (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0'))) { + spec->beyondMax = 1; + } + } +#endif + spec->number += (unsigned long)((SecUnsignedInt)ch - SECUREC_CHAR('0')); + +} + + +/* + * Parse Hex character to integer for 32bit . + */ +static void SecDecodeNumberHex(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_LONG_HEX_BEYOND_MAX(spec->number)) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_SIXTEEN(spec->number); + spec->number += (unsigned long)(unsigned int)SecHexValueOfChar(ch); +} + + +/* + * Parse Octal character to integer for 32bit . + */ +static void SecDecodeNumberOctal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_LONG_OCTAL_BEYOND_MAX(spec->number)) { + spec->beyondMax = 1; + } +#endif + spec->number = SECUREC_MUL_EIGHT(spec->number); + spec->number += (unsigned long)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) +/* Compatible with integer negative values other than int */ +static void SecFinishNumberNegativeOther(int comChr, int numberWidth, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { + if (spec->number > (unsigned long)(1ULL << (SECUREC_LONG_BIT_NUM - 1))) { + spec->number = (unsigned long)(1ULL << (SECUREC_LONG_BIT_NUM - 1)); + } else { + spec->number = (unsigned long)(-(long)spec->number); + } + if (spec->beyondMax != 0) { + if (numberWidth < SECUREC_NUM_WIDTH_INT) { + spec->number = 0; + } else if (numberWidth == SECUREC_NUM_WIDTH_LONG) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1))); + } + } + } else { /* o, u, x, X, p */ + spec->number = (unsigned long)(-(long)spec->number); + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} +/* Compatible processing of integer negative numbers */ +static void SecFinishNumberNegativeInt(int comChr, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + if ((spec->number > SECUREC_MIN_64BITS_NEG_VALUE)) { + spec->number = 0; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + if ((spec->number > SECUREC_MIN_32BITS_NEG_VALUE)) { + spec->number = SECUREC_MIN_32BITS_NEG_VALUE; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } + } +#endif + if (spec->beyondMax != 0) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + spec->number = 0; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + spec->number = SECUREC_MIN_32BITS_NEG_VALUE; + } +#endif + } + } else { /* o, u, x, X ,p */ +#ifdef SECUREC_ON_64BITS + if (spec->number > SECUREC_MAX_32BITS_VALUE_INC) { + spec->number = SECUREC_MAX_32BITS_VALUE; + } else { + spec->number = (unsigned int)(-(int)spec->number); + } +#else + spec->number = (unsigned int)(-(int)spec->number); +#endif + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} + +/* Compatible with integer positive values other than int */ +static void SecFinishNumberPositiveOther(int comChr, int numberWidth, SecScanSpec *spec) +{ + if (comChr == SECUREC_CHAR('d') || comChr == SECUREC_CHAR('i')) { + if (spec->number > ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1)) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1); + } + if ((spec->beyondMax != 0 && numberWidth < SECUREC_NUM_WIDTH_INT)) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + if (spec->beyondMax != 0 && numberWidth == SECUREC_NUM_WIDTH_LONG) { + spec->number = ((unsigned long)(1UL << (SECUREC_LONG_BIT_NUM - 1)) - 1); + } + } else { + if (spec->beyondMax != 0) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } +} + +/* Compatible processing of integer positive numbers */ +static void SecFinishNumberPositiveInt(int comChr, SecScanSpec *spec) +{ + if ((comChr == SECUREC_CHAR('d')) || (comChr == SECUREC_CHAR('i'))) { +#ifdef SECUREC_ON_64BITS + if (SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + if (spec->number > SECUREC_MAX_64BITS_POS_VALUE) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } + } + if (spec->beyondMax != 0 && SecIsLongBitEqual(SECUREC_LP64_BIT_WIDTH)) { + spec->number |= (unsigned long)SECUREC_MAX_64BITS_VALUE; + } +#else + if (SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + if (spec->number > SECUREC_MAX_32BITS_POS_VALUE) { + spec->number = SECUREC_MAX_32BITS_POS_VALUE; + } + } + if (spec->beyondMax != 0 && SecIsLongBitEqual(SECUREC_LP32_BIT_WIDTH)) { + spec->number = SECUREC_MAX_32BITS_POS_VALUE; + } +#endif + } else { /* o,u,x,X,p */ + if (spec->beyondMax != 0) { + spec->number = SECUREC_MAX_32BITS_VALUE; + } + } +} + +#endif + + +/* + * Parse decimal character to integer for 64bit . + */ +static void SecDecodeNumber64Decimal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number64 > SECUREC_MAX_64BITS_VALUE_DIV_TEN) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_TEN(spec->number64); +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->number64 == SECUREC_MAX_64BITS_VALUE_CUT_LAST_DIGIT) { + SecUnsignedInt64 number64As = (SecUnsignedInt64)SECUREC_MAX_64BITS_VALUE - spec->number64; + if (number64As < (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0'))) { + spec->beyondMax = 1; + } + } +#endif + spec->number64 += (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + +/* + * Parse Hex character to integer for 64bit . + */ +static void SecDecodeNumber64Hex(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_QWORD_HEX_BEYOND_MAX(spec->number64)) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_SIXTEEN(spec->number64); + spec->number64 += (SecUnsignedInt64)(unsigned int)SecHexValueOfChar(ch); + +} + +/* + * Parse Octal character to integer for 64bit . + */ +static void SecDecodeNumber64Octal(SecInt ch, SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (SECUREC_QWORD_OCTAL_BEYOND_MAX(spec->number64)) { + spec->beyondMax = 1; + } +#endif + spec->number64 = SECUREC_MUL_EIGHT(spec->number64); + spec->number64 += (SecUnsignedInt64)((SecUnsignedInt)ch - SECUREC_CHAR('0')); +} + +#define SECUREC_DECODE_NUMBER_FUNC_NUM 2 +/* Function name cannot add address symbol, causing 546 alarm */ +static void (*g_secDecodeNumberHex[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberHex, SecDecodeNumber64Hex }; +static void (*g_secDecodeNumberOctal[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberOctal, SecDecodeNumber64Octal }; +static void (*g_secDecodeNumberDecimal[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecInt ch, SecScanSpec *spec) = \ + { SecDecodeNumberDecimal, SecDecodeNumber64Decimal }; + +/* + * Parse 64-bit integer formatted input, return 0 when ch is a number. + */ +static int SecDecodeNumber(SecInt ch, SecScanSpec *spec) +{ + if (spec->comChr == SECUREC_CHAR('x') || spec->comChr == SECUREC_CHAR('p')) { + if (SECUREC_IS_XDIGIT(ch)) { + (*g_secDecodeNumberHex[spec->isInt64Arg])(ch, spec); + } else { + return -1; + } + return 0; + } + if (!(SECUREC_IS_DIGIT(ch))) { + return -1; + } + if (spec->comChr == SECUREC_CHAR('o')) { + if (ch < SECUREC_CHAR('8')) { + (*g_secDecodeNumberOctal[spec->isInt64Arg])(ch, spec); + } else { + return -1; + } + } else { /* comChr is 'd' */ + (*g_secDecodeNumberDecimal[spec->isInt64Arg])(ch, spec); + } + return 0; +} + + +/* + * Complete the final 32-bit integer formatted input + */ +static void SecFinishNumber(SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->negative != 0) { + if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + SecFinishNumberNegativeInt(spec->oriComChr, spec); + } else { + SecFinishNumberNegativeOther(spec->oriComChr, spec->numberWidth, spec); + } + } else { + if (spec->numberWidth == SECUREC_NUM_WIDTH_INT) { + SecFinishNumberPositiveInt(spec->oriComChr, spec); + } else { + SecFinishNumberPositiveOther(spec->oriComChr, spec->numberWidth, spec); + } + } +#else + if (spec->negative != 0) { +#if defined(__hpux) + if (spec->oriComChr != SECUREC_CHAR('p')) { + spec->number = (unsigned long)(-(long)spec->number); + } +#else + spec->number = (unsigned long)(-(long)spec->number); +#endif + } +#endif + return; +} + +/* + * Complete the final 64-bit integer formatted input + */ +static void SecFinishNumber64(SecScanSpec *spec) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && !(defined(SECUREC_ON_UNIX))) + if (spec->negative != 0) { + if (spec->oriComChr == (SECUREC_CHAR('d')) || (spec->oriComChr == SECUREC_CHAR('i'))) { + if (spec->number64 > SECUREC_MIN_64BITS_NEG_VALUE) { + spec->number64 = SECUREC_MIN_64BITS_NEG_VALUE; + } else { + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + } + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MIN_64BITS_NEG_VALUE; + } + } else { /* o, u, x, X, p */ + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_VALUE; + } + } + } else { + if ((spec->oriComChr == SECUREC_CHAR('d')) || (spec->oriComChr == SECUREC_CHAR('i'))) { + if (spec->number64 > SECUREC_MAX_64BITS_POS_VALUE) { + spec->number64 = SECUREC_MAX_64BITS_POS_VALUE; + } + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_POS_VALUE; + } + } else { + if (spec->beyondMax != 0) { + spec->number64 = SECUREC_MAX_64BITS_VALUE; + } + } + } +#else + if (spec->negative != 0) { +#if defined(__hpux) + if (spec->oriComChr != SECUREC_CHAR('p')) { + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); + } +#else + spec->number64 = (SecUnsignedInt64)(-(SecInt64)spec->number64); +#endif + } +#endif + return; +} +static void (*g_secFinishNumber[SECUREC_DECODE_NUMBER_FUNC_NUM])(SecScanSpec *spec) = \ + { SecFinishNumber, SecFinishNumber64 }; + +#if SECUREC_ENABLE_SCANF_FILE + +/* + * Adjust the pointer position of the file stream + */ +static void SecSeekStream(SecFileStream *stream) +{ + if ((stream->count == 0) && feof(stream->pf)) { + /* file pointer at the end of file, don't need to seek back */ + stream->base[0] = '\0'; + return; + } + /* LSD seek to original position, bug fix 2014 1 21 */ + if (fseek(stream->pf, stream->oriFilePos, SEEK_SET)) { + /* seek failed, ignore it */ + stream->oriFilePos = 0; + return; + } + + if (stream->fileRealRead > 0) { /* LSD bug fix. when file reach to EOF, don't seek back */ +#if (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + int loops; + for (loops = 0; loops < (stream->fileRealRead / SECUREC_BUFFERED_BLOK_SIZE); ++loops) { + if (fread(stream->base, (size_t)1, (size_t)SECUREC_BUFFERED_BLOK_SIZE, + stream->pf) != SECUREC_BUFFERED_BLOK_SIZE) { + break; + } + } + if ((stream->fileRealRead % SECUREC_BUFFERED_BLOK_SIZE) != 0) { + size_t ret = fread(stream->base, (size_t)((unsigned int)stream->fileRealRead % SECUREC_BUFFERED_BLOK_SIZE), + (size_t)1, stream->pf); + if ((ret == 1 || ret == 0) && (ftell(stream->pf) < stream->oriFilePos + stream->fileRealRead)) { + (void)fseek(stream->pf, stream->oriFilePos + stream->fileRealRead, SEEK_SET); + } + } + +#else + /* in linux like system */ + if (fseek(stream->pf, stream->oriFilePos + stream->fileRealRead, SEEK_SET)) { + /* seek failed, ignore it */ + stream->oriFilePos = 0; + } +#endif + } + + return; +} + +/* + * Adjust the pointer position of the file stream and free memory + */ +static void SecAdjustStream(SecFileStream *stream) +{ + if (stream != NULL && (stream->flag & SECUREC_FILE_STREAM_FLAG) && stream->base != NULL) { + SecSeekStream(stream); + SECUREC_FREE(stream->base); + stream->base = NULL; + } + return; +} +#endif + +static void SecSkipSpaceFormat(const SecUnsignedChar **format) +{ + const SecUnsignedChar *fmt = *format; + while (SECUREC_IS_SPACE(*fmt)) { + ++fmt; + } + *format = fmt; +} +#ifndef SECUREC_FOR_WCHAR +/* + * Handling multi-character characters + */ +static int SecDecodeLeadByte(SecInt ch, const SecUnsignedChar **format, SecFileStream *stream, int *counter) +{ +#if SECUREC_HAVE_MBTOWC + char temp[SECUREC_MULTI_BYTE_MAX_LEN]; + const SecUnsignedChar *fmt = *format; + wchar_t tempWChar = L'\0'; + int ch2 = SecGetChar(stream, counter); + if (*fmt == SECUREC_CHAR('\0') || (int)(*fmt) != (ch2)) { + /* LSD in console mode, ungetc twice may cause problem */ + SecUnGetChar(ch2, stream, counter); + SecUnGetChar(ch, stream, counter); + return -1; + } + ++fmt; + if (MB_CUR_MAX >= SECUREC_UTF8_BOM_HEADER_SIZE && + (((unsigned char)ch & SECUREC_UTF8_LEAD_1ST) == SECUREC_UTF8_LEAD_1ST) && + (((unsigned char)ch2 & SECUREC_UTF8_LEAD_2ND) == SECUREC_UTF8_LEAD_2ND)) { + /* this char is very likely to be a UTF-8 char */ + int ch3 = SecGetChar(stream, counter); + temp[0] = (char)ch; + temp[1] = (char)ch2; /* 1 index of second character */ + temp[2] = (char)ch3; /* 2 index of third character */ + temp[3] = '\0'; /* 3 of string terminator position */ + + if (mbtowc(&tempWChar, temp, sizeof(temp)) > 0) { + /* succeed */ + if (*fmt == SECUREC_CHAR('\0') || (int)(*fmt) != (int)ch3) { + SecUnGetChar(ch3, stream, counter); + return -1; + } + ++fmt; + *counter = *counter - 1; + } else { + SecUnGetChar(ch3, stream, counter); + } + } + *counter = *counter - 1; /* only count as one character read */ + *format = fmt; + return 0; +#else + SecUnGetChar(ch, stream, counter); + (void)format; + return -1; +#endif +} +#endif + + + +/* + * Resolving sequence of characters from %[ format + */ +static int SecSetupBracketTable(const SecUnsignedChar **format, SecBracketTable *bracketTable) +{ + const SecUnsignedChar *fmt = *format; + SecUnsignedChar prevChar = 0; + SecUnsignedChar expCh; + SecUnsignedChar last = 0; +#if !(defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + if (*fmt == SECUREC_CHAR('{')) { + return -1; + } +#endif + /* for building "table" data */ + ++fmt; /* skip [ */ + bracketTable->mask = 0; + if (*fmt == SECUREC_CHAR('^')) { + ++fmt; + bracketTable->mask = (unsigned char)0xff; + } + if (*fmt == SECUREC_CHAR(']')) { + prevChar = SECUREC_CHAR(']'); + ++fmt; + SECUREC_BRACKET_SET_BIT(bracketTable->table, SECUREC_CHAR(']')); + } + while (*fmt != SECUREC_CHAR('\0') && *fmt != SECUREC_CHAR(']')) { + expCh = *fmt++; + if (expCh != SECUREC_CHAR('-') || prevChar == 0 || *fmt == SECUREC_CHAR(']')) { + /* normal character */ + prevChar = expCh; + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + } else { + /* for %[a-z] */ + expCh = *fmt++; /* get end of range */ + if (prevChar < expCh) { /* %[a-z] */ + last = expCh; + } else { + prevChar = expCh; +#if (defined(SECUREC_COMPATIBLE_WIN_FORMAT)) + /* %[z-a] */ + last = prevChar; + +#else + SECUREC_BRACKET_SET_BIT(bracketTable->table, SECUREC_CHAR('-')); + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + continue; +#endif + } + /* format %[a-\xff] last is 0xFF, condition (rnch <= last) cause dead loop */ + for (expCh = prevChar; expCh < last; ++expCh) { + SECUREC_BRACKET_SET_BIT(bracketTable->table, expCh); + } + SECUREC_BRACKET_SET_BIT(bracketTable->table, last); + prevChar = 0; + } + } + *format = fmt; + return 0; +} + + +#ifdef SECUREC_FOR_WCHAR +static int SecInputForWchar(SecInt ch, SecScanSpec *spec) +{ + void *endPtr = spec->argPtr; + if (spec->isWChar > 0) { + *(wchar_t UNALIGNED *)endPtr = (wchar_t)ch; + endPtr = (wchar_t *)endPtr + 1; + --spec->arrayWidth; + } else { +#if SECUREC_HAVE_WCTOMB + int temp; + char tmpBuf[SECUREC_MB_LEN + 1]; + SECUREC_MASK_MSVC_CRT_WARNING temp = wctomb(tmpBuf, (wchar_t)ch); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (temp <= 0 || ((size_t)(unsigned int)temp) > sizeof(tmpBuf)) { + /* if wctomb error, then ignore character */ + return 0; + } + if (((size_t)(unsigned int)temp) > spec->arrayWidth) { + return -1; + } + if (memcpy_s(endPtr, spec->arrayWidth, tmpBuf, (size_t)(unsigned int)temp) != EOK) { + return -1; + } + endPtr = (char *)endPtr + temp; + spec->arrayWidth -= (size_t)(unsigned int)temp; +#else + return -1; +#endif + } + spec->argPtr = endPtr; + return 0; +} +#endif + + +#ifndef SECUREC_FOR_WCHAR +static int SecInputForChar(SecInt ch, SecScanSpec *spec, SecFileStream *stream, int *charCount) +{ + void *endPtr = spec->argPtr; + if (spec->isWChar > 0) { + wchar_t tempWChar = L'?'; /* set default char as ? */ +#if SECUREC_HAVE_MBTOWC + char temp[SECUREC_MULTI_BYTE_MAX_LEN + 1]; + temp[0] = (char)ch; + temp[1] = '\0'; +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + if (SecIsLeadByte(ch)) { + temp[1] = (char)SecGetChar(stream, charCount); + temp[2] = '\0'; /* 2 of string terminator position */ + } + if (mbtowc(&tempWChar, temp, sizeof(temp)) <= 0) { + /* no string termination error for tool */ + tempWChar = L'?'; + } +#else + if (SecIsLeadByte(ch)) { + int convRes = 0; + int di = 1; + /* in Linux like system, the string is encoded in UTF-8 */ + while (convRes <= 0 && di < (int)MB_CUR_MAX && di < SECUREC_MULTI_BYTE_MAX_LEN) { + temp[di++] = (char)SecGetChar(stream, charCount); + temp[di] = '\0'; + convRes = mbtowc(&tempWChar, temp, sizeof(temp)); + } + if (convRes <= 0) { + tempWChar = L'?'; + } + } else { + if (mbtowc(&tempWChar, temp, sizeof(temp)) <= 0) { + /* no string termination error for tool */ + tempWChar = L'?'; + } + } +#endif +#endif /* SECUREC_HAVE_MBTOWC */ + *(wchar_t UNALIGNED *)endPtr = tempWChar; + /* just copy L'?' if mbtowc fails, errno is set by mbtowc */ + endPtr = (wchar_t *)endPtr + 1; + --spec->arrayWidth; + (void)charCount; + (void)stream; + } else { + *(char *)endPtr = (char)ch; + endPtr = (char *)endPtr + 1; + --spec->arrayWidth; + } + spec->argPtr = endPtr; + return 0; +} +#endif + + +#if SECUREC_ENABLE_SCANF_FLOAT + +/* no not use localeconv()->decimal_pointif onlay support '.' */ +#define SECURE_IS_FLOAT_DECIMAL(ch) ((ch) == SECUREC_CHAR('.')) +/* + * init SecFloatSpec befor parse format + */ +static void SecInitFloatSpec(SecFloatSpec *floatSpec) +{ + floatSpec->floatStr = floatSpec->buffer; + floatSpec->allocatedFloatStr = NULL; + floatSpec->floatStrSize = sizeof(floatSpec->buffer) / sizeof(floatSpec->buffer[0]); + floatSpec->floatStr = floatSpec->buffer; + floatSpec->floatStrUsedLen = 0; +} + +static void SecClearFloatSpec(SecFloatSpec *floatSpec, int *doneCount) +{ + /* LSD 2014.3.6 add, clear the stack data */ + if (memset_s(floatSpec->buffer, sizeof(floatSpec->buffer), 0, + sizeof(floatSpec->buffer)) != EOK) { + *doneCount = 0; /* This is a dead code, just to meet the coding requirements */ + } + if (floatSpec->allocatedFloatStr != NULL) { + /* pFloatStr can be alloced in SecUpdateFloatString function, clear and free it */ + if (memset_s(floatSpec->allocatedFloatStr, floatSpec->floatStrSize * sizeof(SecChar), 0, + floatSpec->floatStrSize * sizeof(SecChar)) != EOK) { + *doneCount = 0; /* This is a dead code, just to meet the coding requirements */ + } + SECUREC_FREE(floatSpec->allocatedFloatStr); + floatSpec->allocatedFloatStr = NULL; + floatSpec->floatStr = NULL; + } +} + + +/* + * scan value of exponent. + * return 0 OK + */ +static int SecInputFloatE(SecFileStream *stream, SecScanSpec *spec, SecFloatSpec *floatSpec, int *charCount) +{ + SecInt ch = SecGetChar(stream, charCount); + if (ch == SECUREC_CHAR('+') || ch == SECUREC_CHAR('-')) { + if (ch == SECUREC_CHAR('-') && SecUpdateFloatString((SecChar)'-', floatSpec) != 0) { + return -1; + } + if (spec->width != 0) { + ch = SecGetChar(stream, charCount); + --spec->width; + } + } + + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + return 0; +} + +/* + * scan %f. + * return 0 OK + */ +static int SecInputFloat(SecFileStream *stream, SecScanSpec *spec, SecFloatSpec *floatSpec, int *charCount) +{ + int started = -1; + SecInt ch = SecGetChar(stream, charCount); + + floatSpec->floatStrUsedLen = 0; + if (ch == SECUREC_CHAR('-')) { + floatSpec->floatStr[floatSpec->floatStrUsedLen++] = SECUREC_CHAR('-'); + --spec->width; + ch = SecGetChar(stream, charCount); + } else if (ch == SECUREC_CHAR('+')) { + --spec->width; + ch = SecGetChar(stream, charCount); + } + + if (spec->widthSet == 0) { /* must care width */ + spec->width = -1; /* -1 is unlimited */ + } + + /* now get integral part */ + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + started = 0; + /* ch must be '0' - '9' */ + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + + /* now get fractional part */ + if (SECURE_IS_FLOAT_DECIMAL((SecChar)ch) && spec->width-- != 0) { + /* now check for decimal */ + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + while (SECUREC_IS_DIGIT(ch) && spec->width-- != 0) { + started = 0; + if (SecUpdateFloatString((SecChar)ch, floatSpec) != 0) { + return -1; + } + ch = SecGetChar(stream, charCount); + } + } + + /* now get exponent part */ + if (started == 0 && (ch == SECUREC_CHAR('e') || ch == SECUREC_CHAR('E')) && spec->width-- != 0) { + if (SecUpdateFloatString((SecChar)'e', floatSpec) != 0) { + return -1; + } + if (SecInputFloatE(stream, spec, floatSpec, charCount) != 0) { + return -1; + } + } + /* un set the last character that is not a floating point number */ + SecUnGetChar(ch, stream, charCount); + /* Make sure have a string terminator, buffer is large enough */ + floatSpec->floatStr[floatSpec->floatStrUsedLen] = SECUREC_CHAR('\0'); + return started; + +} +#endif + +/* + * scan digital part of %d %i %o %u %x %p. + * return 0 OK + */ +static int SecInputNumberDigital(SecInt firstCh, SecFileStream *stream, SecScanSpec *spec, int *charCount) +{ + SecInt ch = firstCh; + int loopFlag = 0; + int started = -1; + while (loopFlag == 0) { + /* decode ch to number */ + loopFlag = SecDecodeNumber(ch, spec); + if (loopFlag == 0) { + started = 0; + if (spec->widthSet != 0 && --spec->width == 0) { + loopFlag = 1; + } else { + ch = SecGetChar(stream, charCount); + } + } else { + SecUnGetChar(ch, stream, charCount); + } + } + + /* Handling integer negative numbers and beyond max */ + (*g_secFinishNumber[spec->isInt64Arg])(spec); + return started; + +} + +/* + * scan %d %i %o %u %x %p. + * return 0 OK + */ +static int SecInputNumber(SecFileStream *stream, SecScanSpec *spec, int *charCount) +{ + SecInt ch = SecGetChar(stream, charCount); + + if (ch == SECUREC_CHAR('+') || ch == SECUREC_CHAR('-')) { + if (ch == SECUREC_CHAR('-')) { + spec->negative = 1; + } + if (spec->widthSet != 0 && --spec->width == 0) { + return -1; + } else { + ch = SecGetChar(stream, charCount); + } + } + + if (spec->oriComChr == SECUREC_CHAR('i')) { + /* i could be d, o, or x, use d as default */ + spec->comChr = SECUREC_CHAR('d'); + } + + if (spec->oriComChr == SECUREC_CHAR('x') || spec->oriComChr == SECUREC_CHAR('i')) { + if (ch != SECUREC_CHAR('0')) { + /* scan number */ + return SecInputNumberDigital(ch, stream, spec, charCount); + } + /* now input string may be 0x123 or 0X123 or just 0 */ + /* get next char */ + ch = SecGetChar(stream, charCount); + if ((SecChar)(ch) == SECUREC_CHAR('x') || (SecChar)ch == SECUREC_CHAR('X')) { + spec->comChr = SECUREC_CHAR('x'); + ch = SecGetChar(stream, charCount); + /* length of 0x is 2 */ + if (spec->widthSet != 0 && spec->width <= (1 + 1)) { + /* length not enough for "0x" */ + return -1; + } + spec->width -= 2; /* Subtract 2 for the length of "0x" */ + } else { + if (spec->oriComChr != SECUREC_CHAR('x')) { + spec->comChr = SECUREC_CHAR('o'); + } + /* unset the character after 0 back to stream, input only '0' result is OK */ + SecUnGetChar(ch, stream, charCount); + ch = SECUREC_CHAR('0'); + } + } + /* scan number */ + return SecInputNumberDigital(ch, stream, spec, charCount); +} +/* + * scan %c %s %[ + * return 0 OK + */ +static int SecInputString(SecFileStream *stream, SecScanSpec *spec, + const SecBracketTable *bracketTable, int *charCount, int *doneCount) +{ + void *startPtr = spec->argPtr; + int suppressed= 0; + int errNoMem = 0; + + while (spec->widthSet == 0 || spec->width-- != 0) { + SecInt ch = SecGetChar(stream, charCount); + /* char condition or string condition and bracket condition. + * only supports wide characters with a maximum length of two bytes + */ + if ((ch != SECUREC_EOF) && (spec->comChr == SECUREC_CHAR('c') || + SECUREC_SCANF_STRING_CONDITION(spec->comChr, ch) || + SECUREC_SCANF_BRACKET_CONDITION(spec->comChr, ch, bracketTable->table, bracketTable->mask))) { + if (spec->suppress != 0) { + /* Used to identify processed data for %* + * use endPtr to identify will cause 613, so use suppressed + */ + suppressed = 1; + continue; + } + /* now suppress is not set */ + if (spec->arrayWidth == 0) { + errNoMem = 1; /* We have exhausted the user's buffer */ + break; + } +#ifdef SECUREC_FOR_WCHAR + errNoMem = SecInputForWchar(ch, spec); +#else + errNoMem = SecInputForChar(ch, spec, stream, charCount); +#endif + if (errNoMem != 0) { + break; + } + } else { + SecUnGetChar(ch, stream, charCount); + break; + } + } + + if (errNoMem != 0) { + /* In case of error, blank out the input buffer */ + if (spec->suppress == 0) { + SecAddEndingZero(startPtr, spec); + } + return -1; + } + + /* No input was scanned */ + if ((spec->suppress != 0 && suppressed == 0) || + (spec->suppress == 0 && startPtr == spec->argPtr)) { + return -1; + } + + if (spec->suppress == 0) { + if (spec->comChr != 'c') { + /* null-terminate strings */ + SecAddEndingZero(spec->argPtr, spec); + } + *doneCount = *doneCount + 1; + } + return 0; +} + +#ifdef SECUREC_FOR_WCHAR +/* + * alloce buffer for wchar version of %[. + * return 0 OK + */ +static int SecAllocBracketTable(SecBracketTable *bracketTable) +{ + if (bracketTable->table == NULL) { + /* table should be freed after use */ + bracketTable->table = (unsigned char *)SECUREC_MALLOC(SECUREC_BRACKET_TABLE_SIZE); + if (bracketTable->table == NULL) { + return -1; + } + } + return 0; +} + +/* + * free buffer for wchar version of %[ + */ +static void SecFreeBracketTable(SecBracketTable *bracketTable) +{ + if (bracketTable->table != NULL) { + SECUREC_FREE(bracketTable->table); + bracketTable->table = NULL; + } +} +#endif + +#ifdef SECUREC_FOR_WCHAR +/* + * Formatting input core functions for wchar version.Called by a function such as vsscanf_s + */ +int SecInputSW(SecFileStream *stream, const wchar_t *cFormat, va_list argList) +#else +/* + * Formatting input core functions for char version.Called by a function such as vswscanf_s + */ +int SecInputS(SecFileStream *stream, const char *cFormat, va_list argList) +#endif +{ + const SecUnsignedChar *format = (const SecUnsignedChar *)cFormat; + SecBracketTable bracketTable = SECUREC_INIT_BRACKET_TABLE; + SecScanSpec spec; + SecInt ch = 0; + int charCount = 0; + int doneCount = 0; + int formatError = 0; + int paraIsNull = 0; +#if SECUREC_ENABLE_SCANF_FLOAT + SecFloatSpec floatSpec; +#endif + int match = 0; + int errRet = 0; +#if SECUREC_ENABLE_SCANF_FLOAT + SecInitFloatSpec(&floatSpec); +#endif + /* format must not NULL */ + /* use err < 1 to claer 845 */ + while (errRet < 1 && *format != SECUREC_CHAR('\0')) { + /* skip space in format and space in input */ + if (SECUREC_IS_SPACE(*format)) { + SecInt nonSpaceChar = SecSkipSpaceChar(stream, &charCount); + /* eat all space chars and put fist no space char backup */ + SecUnGetChar(nonSpaceChar, stream, &charCount); + SecSkipSpaceFormat(&format); + continue; + } + + if (*format != SECUREC_CHAR('%')) { + ch = SecGetChar(stream, &charCount); + if ((int)(*format++) != (int)(ch)) { + SecUnGetChar(ch, stream, &charCount); + ++errRet; /* use plus to clear 845 */ + continue; + } +#ifndef SECUREC_FOR_WCHAR + if (SecIsLeadByte(ch) && SecDecodeLeadByte(ch, &format, stream, &charCount) != 0) { + ++errRet; + continue; + } +#endif + /* for next %n */ + if ((ch == SECUREC_EOF) && ((*format != SECUREC_CHAR('%')) || (*(format + 1) != SECUREC_CHAR('n')))) { + break; + } + continue; + } + + /* now *format is % */ + /* set default value for each % */ + SecSetDefaultScanSpec(&spec); + if (SecDecodeScanFlag(&format, &spec) != 0) { + formatError = 1; + ++errRet; + continue; + } + /* update wchar flag for %S %C */ + SecUpdateWcharFlagByType(*format, &spec); + +#if SECUREC_HAVE_WCHART == 0 + /* in kernel not support wide char */ + if (spec.isWChar > 0) { + formatError = 1; + ++errRet; + continue; + } +#endif + if (spec.widthSet != 0 && spec.width == 0) { + /* 0 width in format */ + ++errRet; + continue; + } + + spec.comChr = (unsigned char)(*format) | (SECUREC_CHAR('a') - SECUREC_CHAR('A')); /* to lowercase */ + spec.oriComChr = spec.comChr; + + if (spec.comChr != SECUREC_CHAR('n')) { + if (spec.comChr != SECUREC_CHAR('c') && spec.comChr != SECUREC_BRACE) { + ch = SecSkipSpaceChar(stream, &charCount); + } else { + ch = SecGetChar(stream, &charCount); + } + if (ch == SECUREC_EOF) { + ++errRet; + continue; + } + } + + /* now no 0 width in format and get one char from input */ + switch (spec.comChr) { + case SECUREC_CHAR('c'): /* also 'C' */ + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('s'): /* also 'S': */ + /* fall-through */ /* FALLTHRU */ + case SECUREC_BRACE: + /* check dest buffer and size */ + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + /* Get the next argument - size of the array in characters */ +#ifdef SECUREC_ON_64BITS + spec.arrayWidth = ((size_t)(va_arg(argList, size_t))) & 0xFFFFFFFFUL; +#else /* !SECUREC_ON_64BITS */ + spec.arrayWidth = (size_t)va_arg(argList, size_t); +#endif + if (spec.arrayWidth == 0 || (spec.isWChar <= 0 && spec.arrayWidth > SECUREC_STRING_MAX_LEN) || + (spec.isWChar > 0 && spec.arrayWidth > SECUREC_WCHAR_STRING_MAX_LEN)) { + /* do not clear buffer just go error */ + ++errRet; + continue; + } + /* One element is needed for '\0' for %s and %[ */ + if (spec.comChr != SECUREC_CHAR('c')) { + --spec.arrayWidth; + } + } else { + /* Set argPtr to NULL is necessary, in supress mode we don't use argPtr to store data */ + spec.argPtr = NULL; + } + + if (spec.comChr == 'c') { + if (spec.widthSet == 0) { + spec.widthSet = 1; + spec.width = 1; + } + } else if (spec.comChr == SECUREC_BRACE) { + /* malloc when first %[ is meet for wchar version */ +#ifdef SECUREC_FOR_WCHAR + if (SecAllocBracketTable(&bracketTable) != 0) { + ++errRet; + continue; + } + +#endif + (void)memset(bracketTable.table, 0, (size_t)SECUREC_BRACKET_TABLE_SIZE); + if (SecSetupBracketTable(&format, &bracketTable) != 0) { + ++errRet; + continue; + } + + if (*format == SECUREC_CHAR('\0')) { + if (spec.suppress == 0 && spec.arrayWidth > 0) { + SecAddEndingZero(spec.argPtr, &spec); + } + ++errRet; + /* truncated format */ + continue; + } + + } + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + /* scanset completed. Now read string */ + if (SecInputString(stream, &spec, &bracketTable, &charCount, &doneCount) != 0) { + ++errRet; + continue; + } + break; + case SECUREC_CHAR('p'): + /* make %hp same as %p */ + spec.numberWidth = SECUREC_NUM_WIDTH_INT; +#ifdef SECUREC_ON_64BITS + spec.isInt64Arg = 1; +#endif + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('o'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('x'): + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + if (SecInputNumber(stream, &spec, &charCount) != 0) { + ++errRet; + continue; + } + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + SecAssignNumber(&spec); + ++doneCount; + } + break; + case SECUREC_CHAR('n'): /* char count */ + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + paraIsNull = 1; + ++errRet; + continue; + } + spec.number = (unsigned long)(unsigned int)charCount; + spec.isInt64Arg = 0; + SecAssignNumber(&spec); + } + break; + case SECUREC_CHAR('e'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('f'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('g'): /* scan a float */ +#if SECUREC_ENABLE_SCANF_FLOAT + /* un set last char to stream */ + SecUnGetChar(ch, stream, &charCount); + if (SecInputFloat(stream, &spec, &floatSpec, &charCount) != 0) { + ++errRet; + continue; + } + if (spec.suppress == 0) { + spec.argPtr = (void *)va_arg(argList, void *); + if (spec.argPtr == NULL) { + ++errRet; + paraIsNull = 1; + continue; + } +#ifdef SECUREC_FOR_WCHAR + if (SecAssignFloatW(&floatSpec, &spec) != 0) { + ++errRet; + continue; + } +#else + SecAssignFloat(floatSpec.floatStr, spec.numberWidth, spec.argPtr); +#endif + ++doneCount; + } + + break; +#else /* SECUREC_ENABLE_SCANF_FLOAT */ + ++errRet; + continue; +#endif + default: + if ((int)(*format) != (int)ch) { + SecUnGetChar(ch, stream, &charCount); + formatError = 1; + ++errRet; + continue; + } else { + --match; + } + } + + ++match; + ++format; + if ((ch == SECUREC_EOF) && ((*format != SECUREC_CHAR('%')) || (*(format + 1) != SECUREC_CHAR('n')))) { + break; + } + } + +#ifdef SECUREC_FOR_WCHAR + SecFreeBracketTable(&bracketTable); +#endif + +#if SECUREC_ENABLE_SCANF_FLOAT + SecClearFloatSpec(&floatSpec, &doneCount); +#endif + +#if SECUREC_ENABLE_SCANF_FILE + SecAdjustStream(stream); +#endif + + if (ch == SECUREC_EOF) { + return ((doneCount || match) ? doneCount : SECUREC_SCANF_EINVAL); + } else if (formatError != 0 || paraIsNull != 0) { + /* Invalid Input Format or parameter */ + return SECUREC_SCANF_ERROR_PARA; + } + + return doneCount; +} + +#if SECUREC_ENABLE_SCANF_FILE + +#if defined(SECUREC_NO_STD_UNGETC) +/* + * Get char from stdin or buffer + */ +static SecInt SecGetCharFromStdin(SecFileStream *stream) +{ + SecInt ch; + if (stream->fUnget == 1) { + ch = (SecInt) stream->lastChar; + stream->fUnget = 0; + } else { + ch = SECUREC_GETC(stream->pf); + stream->lastChar = (unsigned int)ch; + } + return ch; +} +#else +/* + * Get char from stdin or buffer use std function + */ +static SecInt SecGetCharFromStdin(const SecFileStream *stream) +{ + SecInt ch; + ch = SECUREC_GETC(stream->pf); + return ch; +} +#endif + +static void SecSkipBomHeader(SecFileStream *stream) +{ +#ifdef SECUREC_FOR_WCHAR + if (stream->count >= SECUREC_BOM_HEADER_SIZE && + (((unsigned char)(stream->base[0]) == SECUREC_BOM_HEADER_LE_1ST && + (unsigned char)(stream->base[1]) == SECUREC_BOM_HEADER_LE_2ST) || + ((unsigned char)(stream->base[0]) == SECUREC_BOM_HEADER_BE_1ST && + (unsigned char)(stream->base[1]) == SECUREC_BOM_HEADER_BE_2ST))) { + + /* the stream->count must be a multiple of sizeof(SecChar), + * otherwise this function will return SECUREC_EOF when read the last character + */ + if ((stream->count - SECUREC_BOM_HEADER_SIZE) % (int)sizeof(SecChar) != 0) { + int ret = (int)fread(stream->base + stream->count, (size_t)1, + (size_t)SECUREC_BOM_HEADER_SIZE, stream->pf); + if (ret > 0 && ret <= SECUREC_BUFFERED_BLOK_SIZE) { + stream->count += ret; + } + } + /* it's BOM header, skip */ + stream->count -= SECUREC_BOM_HEADER_SIZE; + stream->cur += SECUREC_BOM_HEADER_SIZE; + } +#else + if (stream->count >= SECUREC_UTF8_BOM_HEADER_SIZE && + (unsigned char)(stream->base[0]) == SECUREC_UTF8_BOM_HEADER_1ST && + (unsigned char)(stream->base[1]) == SECUREC_UTF8_BOM_HEADER_2ND && + (unsigned char)(stream->base[2]) == SECUREC_UTF8_BOM_HEADER_3RD) { /* 2 offset of third head character */ + /* it's BOM header, skip */ + stream->count -= SECUREC_UTF8_BOM_HEADER_SIZE; + stream->cur += SECUREC_UTF8_BOM_HEADER_SIZE; + } +#endif +} +/* + * Get char from file stream or buffer + */ +static SecInt SecGetCharFromFile(SecFileStream *stream) +{ + SecInt ch; + if (stream->count == 0) { + int firstReadOnFile = 0; + /* load file to buffer */ + if (stream->base == NULL) { + stream->base = (char *)SECUREC_MALLOC(SECUREC_BUFFERED_BLOK_SIZE + 1); + if (stream->base == NULL) { + return SECUREC_EOF; + } + stream->base[SECUREC_BUFFERED_BLOK_SIZE] = '\0'; /* for tool Warning string null */ + } + /* LSD add 2014.3.21 */ + if (stream->oriFilePos == SECUREC_UNINITIALIZED_FILE_POS) { + stream->oriFilePos = ftell(stream->pf); /* save original file read position */ + firstReadOnFile = 1; + } + stream->count = (int)fread(stream->base, (size_t)1, (size_t)SECUREC_BUFFERED_BLOK_SIZE, stream->pf); + stream->base[SECUREC_BUFFERED_BLOK_SIZE] = '\0'; /* for tool Warning string null */ + if (stream->count == 0 || stream->count > SECUREC_BUFFERED_BLOK_SIZE) { + return SECUREC_EOF; + } + stream->cur = stream->base; + stream->flag |= SECUREC_LOAD_FILE_TO_MEM_FLAG; + if (firstReadOnFile != 0) { + SecSkipBomHeader(stream); + } + } + /* according wchar_t has two bytes */ + ch = (SecInt)((stream->count -= (int)sizeof(SecChar)) >= 0 ? \ + (SecInt)(SECUREC_CHAR_MASK & \ + (unsigned int)(int)(*((const SecChar *)(const void *)stream->cur))) : SECUREC_EOF); + stream->cur += sizeof(SecChar); + + if (ch != SECUREC_EOF && stream->base != NULL) { + stream->fileRealRead += (int)sizeof(SecChar); + } + return ch; +} +#endif + +/* + * Get char for wchar version + */ +static SecInt SecGetChar(SecFileStream *stream, int *counter) +{ + SecInt ch = SECUREC_EOF; +#if SECUREC_ENABLE_SCANF_FILE + if ((stream->flag & SECUREC_FROM_STDIN_FLAG) > 0) { + ch = SecGetCharFromStdin(stream); + } else if ((stream->flag & SECUREC_FILE_STREAM_FLAG) > 0) { + ch = SecGetCharFromFile(stream); + } +#endif + if ((stream->flag & SECUREC_MEM_STR_FLAG) > 0) { + /* according wchar_t has two bytes */ + ch = (SecInt)((stream->count -= (int)sizeof(SecChar)) >= 0 ? \ + (SecInt)(SECUREC_CHAR_MASK & \ + (unsigned int)(int)(*((const SecChar *)(const void *)stream->cur))) : SECUREC_EOF); + stream->cur += sizeof(SecChar); + } + *counter = *counter + 1; + return ch; +} + +/* + * Unget Public realizatio char for wchar and char version + */ +static void SecUnGetCharImpl(SecInt ch, SecFileStream *stream) +{ + if ((stream->flag & SECUREC_FROM_STDIN_FLAG) > 0) { +#if SECUREC_ENABLE_SCANF_FILE +#if defined(SECUREC_NO_STD_UNGETC) + stream->lastChar = (unsigned int)ch; + stream->fUnget = 1; +#else + (void)SECUREC_UN_GETC(ch, stream->pf); +#endif +#else + (void)ch; /* to clear e438 last value assigned not used , the compiler will optimize this code */ +#endif + } else if ((stream->flag & SECUREC_MEM_STR_FLAG) || (stream->flag & SECUREC_LOAD_FILE_TO_MEM_FLAG) > 0) { + if (stream->cur > stream->base) { + stream->cur -= sizeof(SecChar); + stream->count += (int)sizeof(SecChar); + } + } +#if SECUREC_ENABLE_SCANF_FILE + if ((stream->flag & SECUREC_FILE_STREAM_FLAG) > 0 && stream->base) { + stream->fileRealRead -= (int)sizeof(SecChar); + } +#endif +} + +/* + * Unget char for char version + */ +static void SecUnGetChar(SecInt ch, SecFileStream *stream, int *counter) +{ + if (ch != SECUREC_EOF) { + SecUnGetCharImpl(ch, stream); + } + *counter = *counter - 1; +} + +/* + * Skip space char by isspace + */ +static SecInt SecSkipSpaceChar(SecFileStream *stream, int *counter) +{ + SecInt ch; + do { + ch = SecGetChar(stream, counter); + } while (ch != SECUREC_EOF && SECUREC_IS_SPACE(ch)); + return ch; +} +#endif /* __INPUT_INL__5D13A042_DC3F_4ED9_A8D1_882811274C27 */ + diff --git a/third_party/securec/src/memcpy_s.c b/third_party/securec/src/memcpy_s.c new file mode 100644 index 00000000..6d350581 --- /dev/null +++ b/third_party/securec/src/memcpy_s.c @@ -0,0 +1,576 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 +#include "securecutil.h" + +#ifndef SECUREC_MEMCOPY_WITH_PERFORMANCE +#define SECUREC_MEMCOPY_WITH_PERFORMANCE 0 +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMCOPY_WITH_PERFORMANCE +#ifndef SECUREC_MEMCOPY_THRESHOLD_SIZE +#define SECUREC_MEMCOPY_THRESHOLD_SIZE 64UL +#endif +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +#define SECUREC_SMALL_MEM_COPY do { \ + if (SECUREC_ADDR_ALIGNED_8(dest) && SECUREC_ADDR_ALIGNED_8(src)) { \ + /* use struct assignment */ \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)src; \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)src; \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)src; \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)src; \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)src; \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)src; \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)src; \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)src; \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)src; \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)src; \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)src; \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)src; \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)src; \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)src; \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)src; \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)src; \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)src; \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)src; \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)src; \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)src; \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)src; \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)src; \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)src; \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)src; \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)src; \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)src; \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)src; \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)src; \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)src; \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)src; \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)src; \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)src; \ + break; \ + case 33: \ + *(SecStrBuf33 *)dest = *(const SecStrBuf33 *)src; \ + break; \ + case 34: \ + *(SecStrBuf34 *)dest = *(const SecStrBuf34 *)src; \ + break; \ + case 35: \ + *(SecStrBuf35 *)dest = *(const SecStrBuf35 *)src; \ + break; \ + case 36: \ + *(SecStrBuf36 *)dest = *(const SecStrBuf36 *)src; \ + break; \ + case 37: \ + *(SecStrBuf37 *)dest = *(const SecStrBuf37 *)src; \ + break; \ + case 38: \ + *(SecStrBuf38 *)dest = *(const SecStrBuf38 *)src; \ + break; \ + case 39: \ + *(SecStrBuf39 *)dest = *(const SecStrBuf39 *)src; \ + break; \ + case 40: \ + *(SecStrBuf40 *)dest = *(const SecStrBuf40 *)src; \ + break; \ + case 41: \ + *(SecStrBuf41 *)dest = *(const SecStrBuf41 *)src; \ + break; \ + case 42: \ + *(SecStrBuf42 *)dest = *(const SecStrBuf42 *)src; \ + break; \ + case 43: \ + *(SecStrBuf43 *)dest = *(const SecStrBuf43 *)src; \ + break; \ + case 44: \ + *(SecStrBuf44 *)dest = *(const SecStrBuf44 *)src; \ + break; \ + case 45: \ + *(SecStrBuf45 *)dest = *(const SecStrBuf45 *)src; \ + break; \ + case 46: \ + *(SecStrBuf46 *)dest = *(const SecStrBuf46 *)src; \ + break; \ + case 47: \ + *(SecStrBuf47 *)dest = *(const SecStrBuf47 *)src; \ + break; \ + case 48: \ + *(SecStrBuf48 *)dest = *(const SecStrBuf48 *)src; \ + break; \ + case 49: \ + *(SecStrBuf49 *)dest = *(const SecStrBuf49 *)src; \ + break; \ + case 50: \ + *(SecStrBuf50 *)dest = *(const SecStrBuf50 *)src; \ + break; \ + case 51: \ + *(SecStrBuf51 *)dest = *(const SecStrBuf51 *)src; \ + break; \ + case 52: \ + *(SecStrBuf52 *)dest = *(const SecStrBuf52 *)src; \ + break; \ + case 53: \ + *(SecStrBuf53 *)dest = *(const SecStrBuf53 *)src; \ + break; \ + case 54: \ + *(SecStrBuf54 *)dest = *(const SecStrBuf54 *)src; \ + break; \ + case 55: \ + *(SecStrBuf55 *)dest = *(const SecStrBuf55 *)src; \ + break; \ + case 56: \ + *(SecStrBuf56 *)dest = *(const SecStrBuf56 *)src; \ + break; \ + case 57: \ + *(SecStrBuf57 *)dest = *(const SecStrBuf57 *)src; \ + break; \ + case 58: \ + *(SecStrBuf58 *)dest = *(const SecStrBuf58 *)src; \ + break; \ + case 59: \ + *(SecStrBuf59 *)dest = *(const SecStrBuf59 *)src; \ + break; \ + case 60: \ + *(SecStrBuf60 *)dest = *(const SecStrBuf60 *)src; \ + break; \ + case 61: \ + *(SecStrBuf61 *)dest = *(const SecStrBuf61 *)src; \ + break; \ + case 62: \ + *(SecStrBuf62 *)dest = *(const SecStrBuf62 *)src; \ + break; \ + case 63: \ + *(SecStrBuf63 *)dest = *(const SecStrBuf63 *)src; \ + break; \ + case 64: \ + *(SecStrBuf64 *)dest = *(const SecStrBuf64 *)src; \ + break; \ + default: \ + break; \ + } /* END switch */ \ + } else { \ + char *tmpDest = (char *)dest; \ + const char *tmpSrc = (const char *)src; \ + switch (count) { \ + case 64: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 63: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 62: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 61: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 60: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 59: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 58: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 57: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 56: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 55: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 54: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 53: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 52: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 51: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 50: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 49: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 48: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 47: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 46: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 45: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 44: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 43: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 42: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 41: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 40: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 39: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 38: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 37: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 36: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 35: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 34: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 33: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 32: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(tmpDest++) = *(tmpSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +static errno_t SecMemcpyError(void *dest, size_t destMax, const void *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE; + } + if (dest == NULL || src == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memcpy_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax); + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE_AND_RESET; + } + if (dest == src) { + return EOK; + } + if ((dest > src && dest < (const void *)((const unsigned char *)src + count)) || \ + (src > dest && src < (void *)((unsigned char *)dest + count))) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_BUFFER_OVERLAP("memcpy_s"); + return EOVERLAP_AND_RESET; + } + /* count == 0 also return EOK */ + return EOK; +} + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMCOPY_WITH_PERFORMANCE +/* + * Performance optimization + */ +static void SecDoMemcpyOpt(void *dest, const void *src, size_t count) +{ + if (count > SECUREC_MEMCOPY_THRESHOLD_SIZE) { + SecDoMemcpy(dest, src, count); + } else { + SECUREC_SMALL_MEM_COPY; + } + return; +} +#endif + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + /* fread API in windows will call memcpy_s and pass 0xffffffff to destMax. + * To avoid the failure of fread, we don't check desMax limit. + */ +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#else +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && \ + (destMax) <= SECUREC_MEM_MAX_LEN && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#endif + +/* + * + * The memcpy_s function copies n characters from the object pointed to by src into the object pointed to by dest + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Buffer to copy from. + * count Number of characters to copy + * + * + * dest buffer is updated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * and dest != NULL and src != NULL + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and + * count <= destMax destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * and src != NULL and dest != src + * + * if an error occured, dest will be filled with 0. + * If the source and destination overlap, the behavior of memcpy_s is undefined. + * Use memmove_s to handle overlapping regions. + */ +errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { +#if SECUREC_MEMCOPY_WITH_PERFORMANCE + SecDoMemcpyOpt(dest, src, count); +#else + SecDoMemcpy(dest, src, count); +#endif + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memcpy_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memcpy_sOptAsm(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { + SecDoMemcpyOpt(dest, src, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +/* trim judgement on "destMax <= SECUREC_MEM_MAX_LEN" */ +errno_t memcpy_sOptTc(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL && src != NULL && \ + count > 0 && \ + ((dest > src && (const void *)((const unsigned char *)src + count) <= dest) || \ + (src > dest && (void *)((unsigned char *)dest + count) <= src)))) { + SecDoMemcpyOpt(dest, src, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} +#endif + diff --git a/third_party/securec/src/memmove_s.c b/third_party/securec/src/memmove_s.c new file mode 100644 index 00000000..683aed83 --- /dev/null +++ b/third_party/securec/src/memmove_s.c @@ -0,0 +1,119 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securecutil.h" + +#ifdef SECUREC_NOT_CALL_LIBC_CORE_API +/* + * Implementing memory data movement + */ +static void SecUtilMemmove(void *dst, const void *src, size_t count) +{ + unsigned char *pDest = (unsigned char *)dst; + const unsigned char *pSrc = (const unsigned char *)src; + size_t maxCount = count; + + if (dst <= src || pDest >= (pSrc + maxCount)) { + /* + * Non-Overlapping Buffers + * copy from lower addresses to higher addresses + */ + while (maxCount--) { + *pDest = *pSrc; + ++pDest; + ++pSrc; + } + } else { + /* + * Overlapping Buffers + * copy from higher addresses to lower addresses + */ + pDest = pDest + maxCount - 1; + pSrc = pSrc + maxCount - 1; + + while (maxCount--) { + *pDest = *pSrc; + + --pDest; + --pSrc; + } + } +} +#endif + +/* + * + * The memmove_s function copies count bytes of characters from src to dest. + * This function can be assigned correctly when memory overlaps. + * + * dest Destination object. + * destMax Size of the destination buffer. + * src Source object. + * count Number of characters to copy. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > destMax and dest != NULL and src != NULL and destMax != 0 + * and destMax <= SECUREC_MEM_MAX_LEN + * + * If an error occured, dest will be filled with 0 when dest and destMax valid. + * If some regions of the source area and the destination overlap, memmove_s + * ensures that the original source bytes in the overlapping region are copied + * before being overwritten. + */ +errno_t memmove_s(void *dest, size_t destMax, const void *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memmove_s"); + return ERANGE; + } + if (dest == NULL || src == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memmove_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax); + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, 0, destMax); + SECUREC_ERROR_INVALID_RANGE("memmove_s"); + return ERANGE_AND_RESET; + } + if (dest == src) { + return EOK; + } + + if (count > 0) { +#ifdef SECUREC_NOT_CALL_LIBC_CORE_API + SecUtilMemmove(dest, src, count); +#else + /* use underlying memmove for performance consideration */ + (void)memmove(dest, src, count); +#endif + } + return EOK; +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memmove_s); +#endif + diff --git a/third_party/securec/src/memset_s.c b/third_party/securec/src/memset_s.c new file mode 100644 index 00000000..2dd67121 --- /dev/null +++ b/third_party/securec/src/memset_s.c @@ -0,0 +1,521 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMSET 1 + +#include "securecutil.h" + +#ifndef SECUREC_MEMSET_WITH_PERFORMANCE +#define SECUREC_MEMSET_WITH_PERFORMANCE 0 +#endif + +#define SECUREC_MEMSET_PARAM_OK(dest, destMax, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (destMax) <= SECUREC_MEM_MAX_LEN)) + + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMSET_WITH_PERFORMANCE +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +/* use union to clear strict-aliasing warning */ +typedef union { + SecStrBuf32 buf32; + SecStrBuf31 buf31; + SecStrBuf30 buf30; + SecStrBuf29 buf29; + SecStrBuf28 buf28; + SecStrBuf27 buf27; + SecStrBuf26 buf26; + SecStrBuf25 buf25; + SecStrBuf24 buf24; + SecStrBuf23 buf23; + SecStrBuf22 buf22; + SecStrBuf21 buf21; + SecStrBuf20 buf20; + SecStrBuf19 buf19; + SecStrBuf18 buf18; + SecStrBuf17 buf17; + SecStrBuf16 buf16; + SecStrBuf15 buf15; + SecStrBuf14 buf14; + SecStrBuf13 buf13; + SecStrBuf12 buf12; + SecStrBuf11 buf11; + SecStrBuf10 buf10; + SecStrBuf9 buf9; + SecStrBuf8 buf8; + SecStrBuf7 buf7; + SecStrBuf6 buf6; + SecStrBuf5 buf5; + SecStrBuf4 buf4; + SecStrBuf3 buf3; + SecStrBuf2 buf2; + SecStrBuf1 buf1; +} SecStrBuf32Union; +/* C standard initializes the first member of the consortium. */ +static const SecStrBuf32 g_allZero = {{ + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0' +}}; +static const SecStrBuf32 g_allFF = {{ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF +}}; + +static const SecStrBuf32Union *SecStrictAliasingCast(const SecStrBuf32 *buf) +{ + return (const SecStrBuf32Union *)buf; +} + +#ifndef SECUREC_MEMSET_THRESHOLD_SIZE +#define SECUREC_MEMSET_THRESHOLD_SIZE 32UL +#endif + +#define SECUREC_UNALIGNED_SET do { \ + char *pcDest = (char *)dest; \ + switch (count) { \ + case 32: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(pcDest++) = (char)c; \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ +} SECUREC_WHILE_ZERO + +#define SECUREC_ALIGNED_SET_OPT_ZERO_FF do { \ + switch (c) { \ + case 0: \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)(&((SecStrictAliasingCast(&g_allZero))->buf1)); \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)(&((SecStrictAliasingCast(&g_allZero))->buf2)); \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)(&((SecStrictAliasingCast(&g_allZero))->buf3)); \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)(&((SecStrictAliasingCast(&g_allZero))->buf4)); \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)(&((SecStrictAliasingCast(&g_allZero))->buf5)); \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)(&((SecStrictAliasingCast(&g_allZero))->buf6)); \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)(&((SecStrictAliasingCast(&g_allZero))->buf7)); \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)(&((SecStrictAliasingCast(&g_allZero))->buf8)); \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)(&((SecStrictAliasingCast(&g_allZero))->buf9)); \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)(&((SecStrictAliasingCast(&g_allZero))->buf10)); \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)(&((SecStrictAliasingCast(&g_allZero))->buf11)); \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)(&((SecStrictAliasingCast(&g_allZero))->buf12)); \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)(&((SecStrictAliasingCast(&g_allZero))->buf13)); \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)(&((SecStrictAliasingCast(&g_allZero))->buf14)); \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)(&((SecStrictAliasingCast(&g_allZero))->buf15)); \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)(&((SecStrictAliasingCast(&g_allZero))->buf16)); \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)(&((SecStrictAliasingCast(&g_allZero))->buf17)); \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)(&((SecStrictAliasingCast(&g_allZero))->buf18)); \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)(&((SecStrictAliasingCast(&g_allZero))->buf19)); \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)(&((SecStrictAliasingCast(&g_allZero))->buf20)); \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)(&((SecStrictAliasingCast(&g_allZero))->buf21)); \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)(&((SecStrictAliasingCast(&g_allZero))->buf22)); \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)(&((SecStrictAliasingCast(&g_allZero))->buf23)); \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)(&((SecStrictAliasingCast(&g_allZero))->buf24)); \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)(&((SecStrictAliasingCast(&g_allZero))->buf25)); \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)(&((SecStrictAliasingCast(&g_allZero))->buf26)); \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)(&((SecStrictAliasingCast(&g_allZero))->buf27)); \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)(&((SecStrictAliasingCast(&g_allZero))->buf28)); \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)(&((SecStrictAliasingCast(&g_allZero))->buf29)); \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)(&((SecStrictAliasingCast(&g_allZero))->buf30)); \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)(&((SecStrictAliasingCast(&g_allZero))->buf31)); \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)(&((SecStrictAliasingCast(&g_allZero))->buf32)); \ + break; \ + default: \ + break; \ + } \ + break; \ + case 0xFF: \ + switch (count) { \ + case 1: \ + *(SecStrBuf1 *)dest = *(const SecStrBuf1 *)(&((SecStrictAliasingCast(&g_allFF))->buf1)); \ + break; \ + case 2: \ + *(SecStrBuf2 *)dest = *(const SecStrBuf2 *)(&((SecStrictAliasingCast(&g_allFF))->buf2)); \ + break; \ + case 3: \ + *(SecStrBuf3 *)dest = *(const SecStrBuf3 *)(&((SecStrictAliasingCast(&g_allFF))->buf3)); \ + break; \ + case 4: \ + *(SecStrBuf4 *)dest = *(const SecStrBuf4 *)(&((SecStrictAliasingCast(&g_allFF))->buf4)); \ + break; \ + case 5: \ + *(SecStrBuf5 *)dest = *(const SecStrBuf5 *)(&((SecStrictAliasingCast(&g_allFF))->buf5)); \ + break; \ + case 6: \ + *(SecStrBuf6 *)dest = *(const SecStrBuf6 *)(&((SecStrictAliasingCast(&g_allFF))->buf6)); \ + break; \ + case 7: \ + *(SecStrBuf7 *)dest = *(const SecStrBuf7 *)(&((SecStrictAliasingCast(&g_allFF))->buf7)); \ + break; \ + case 8: \ + *(SecStrBuf8 *)dest = *(const SecStrBuf8 *)(&((SecStrictAliasingCast(&g_allFF))->buf8)); \ + break; \ + case 9: \ + *(SecStrBuf9 *)dest = *(const SecStrBuf9 *)(&((SecStrictAliasingCast(&g_allFF))->buf9)); \ + break; \ + case 10: \ + *(SecStrBuf10 *)dest = *(const SecStrBuf10 *)(&((SecStrictAliasingCast(&g_allFF))->buf10)); \ + break; \ + case 11: \ + *(SecStrBuf11 *)dest = *(const SecStrBuf11 *)(&((SecStrictAliasingCast(&g_allFF))->buf11)); \ + break; \ + case 12: \ + *(SecStrBuf12 *)dest = *(const SecStrBuf12 *)(&((SecStrictAliasingCast(&g_allFF))->buf12)); \ + break; \ + case 13: \ + *(SecStrBuf13 *)dest = *(const SecStrBuf13 *)(&((SecStrictAliasingCast(&g_allFF))->buf13)); \ + break; \ + case 14: \ + *(SecStrBuf14 *)dest = *(const SecStrBuf14 *)(&((SecStrictAliasingCast(&g_allFF))->buf14)); \ + break; \ + case 15: \ + *(SecStrBuf15 *)dest = *(const SecStrBuf15 *)(&((SecStrictAliasingCast(&g_allFF))->buf15)); \ + break; \ + case 16: \ + *(SecStrBuf16 *)dest = *(const SecStrBuf16 *)(&((SecStrictAliasingCast(&g_allFF))->buf16)); \ + break; \ + case 17: \ + *(SecStrBuf17 *)dest = *(const SecStrBuf17 *)(&((SecStrictAliasingCast(&g_allFF))->buf17)); \ + break; \ + case 18: \ + *(SecStrBuf18 *)dest = *(const SecStrBuf18 *)(&((SecStrictAliasingCast(&g_allFF))->buf18)); \ + break; \ + case 19: \ + *(SecStrBuf19 *)dest = *(const SecStrBuf19 *)(&((SecStrictAliasingCast(&g_allFF))->buf19)); \ + break; \ + case 20: \ + *(SecStrBuf20 *)dest = *(const SecStrBuf20 *)(&((SecStrictAliasingCast(&g_allFF))->buf20)); \ + break; \ + case 21: \ + *(SecStrBuf21 *)dest = *(const SecStrBuf21 *)(&((SecStrictAliasingCast(&g_allFF))->buf21)); \ + break; \ + case 22: \ + *(SecStrBuf22 *)dest = *(const SecStrBuf22 *)(&((SecStrictAliasingCast(&g_allFF))->buf22)); \ + break; \ + case 23: \ + *(SecStrBuf23 *)dest = *(const SecStrBuf23 *)(&((SecStrictAliasingCast(&g_allFF))->buf23)); \ + break; \ + case 24: \ + *(SecStrBuf24 *)dest = *(const SecStrBuf24 *)(&((SecStrictAliasingCast(&g_allFF))->buf24)); \ + break; \ + case 25: \ + *(SecStrBuf25 *)dest = *(const SecStrBuf25 *)(&((SecStrictAliasingCast(&g_allFF))->buf25)); \ + break; \ + case 26: \ + *(SecStrBuf26 *)dest = *(const SecStrBuf26 *)(&((SecStrictAliasingCast(&g_allFF))->buf26)); \ + break; \ + case 27: \ + *(SecStrBuf27 *)dest = *(const SecStrBuf27 *)(&((SecStrictAliasingCast(&g_allFF))->buf27)); \ + break; \ + case 28: \ + *(SecStrBuf28 *)dest = *(const SecStrBuf28 *)(&((SecStrictAliasingCast(&g_allFF))->buf28)); \ + break; \ + case 29: \ + *(SecStrBuf29 *)dest = *(const SecStrBuf29 *)(&((SecStrictAliasingCast(&g_allFF))->buf29)); \ + break; \ + case 30: \ + *(SecStrBuf30 *)dest = *(const SecStrBuf30 *)(&((SecStrictAliasingCast(&g_allFF))->buf30)); \ + break; \ + case 31: \ + *(SecStrBuf31 *)dest = *(const SecStrBuf31 *)(&((SecStrictAliasingCast(&g_allFF))->buf31)); \ + break; \ + case 32: \ + *(SecStrBuf32 *)dest = *(const SecStrBuf32 *)(&((SecStrictAliasingCast(&g_allFF))->buf32)); \ + break; \ + default: \ + break; \ + } \ + break; \ + default: \ + SECUREC_UNALIGNED_SET; \ + } /* END switch */ \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +static errno_t SecMemsetError(void *dest, size_t destMax, int c, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE; + } + if (dest == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memset_s"); + return EINVAL; + } + if (count > destMax) { + (void)memset(dest, c, destMax); /* set entire buffer to value c */ + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +#if SECUREC_WITH_PERFORMANCE_ADDONS || SECUREC_MEMSET_WITH_PERFORMANCE +/* + * Performance optimization + */ +static void SecDoMemsetOpt(void *dest, int c, size_t count) +{ + if (count > SECUREC_MEMSET_THRESHOLD_SIZE) { + SecDoMemset(dest, c, count); + } else { + if (SECUREC_ADDR_ALIGNED_8(dest)) { + /* use struct assignment */ + SECUREC_ALIGNED_SET_OPT_ZERO_FF; + } else { + SECUREC_UNALIGNED_SET; + } + } + return; +} +#endif + +/* + * + * The memset_s function copies the value of c (converted to an unsigned char) + * into each of the first count characters of the object pointed to by dest. + * + * + * dest Pointer to destination. + * destMax The size of the buffer. + * c Character to set. + * count Number of characters. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest == NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax is 0 or destMax > SECUREC_MEM_MAX_LEN + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * + * if return ERANGE_AND_RESET then fill dest to c ,fill length is destMax + */ +errno_t memset_s(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { +#if SECUREC_MEMSET_WITH_PERFORMANCE + SecDoMemsetOpt(dest, c, count); +#else + SecDoMemset(dest, c, count); +#endif + return EOK; + } else { + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); + } +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(memset_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memset_sOptAsm(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { + SecDoMemsetOpt(dest, c, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); +} + +/* + * Performance optimization + */ +errno_t memset_sOptTc(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL)) { + SecDoMemsetOpt(dest, c, count); + return EOK; + } + /* meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c, count); +} +#endif + diff --git a/third_party/securec/src/output.inl b/third_party/securec/src/output.inl new file mode 100644 index 00000000..b8e0fd65 --- /dev/null +++ b/third_party/securec/src/output.inl @@ -0,0 +1,1400 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 +#define OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 + +#define SECUREC_NULL_STRING_SIZE 8 +#define SECUREC_STATE_TABLE_SIZE 337 +#define SECUREC_OFFSET_BITS_WORD 16 +#define SECUREC_OFFSET_BITS_DWORD 32 + +#define SECUREC_OFFSET_DIV_OCTAL 3 +#define SECUREC_OFFSET_DIV_HEX 4 +#define SECUREC_RADIX_OCTAL 8 +#define SECUREC_RADIX_DECIMAL 10 +#define SECUREC_RADIX_HEX 16 +/* Use two displacements to eliminate compilation warnings */ +#define SECUREC_SHR_DWORD(x) (((x) >> 16) >> 16) +#define SECUREC_PREFIX_LEN 2 +/* size include '+' and '\0' */ +#define SECUREC_FLOAT_BUF_EXT 2 + + +#ifdef SECUREC_STACK_SIZE_LESS_THAN_1K +#define SECUREC_FMT_STR_LEN 8 +#else +#define SECUREC_FMT_STR_LEN 16 +#endif + +typedef struct { + unsigned int flags; + int fldWidth; + int precision; + int bufferIsWide; /* flag for buffer contains wide chars ;0 is not wide char */ + int dynWidth; /* %* 1 width from variable parameter ;0 not */ + int dynPrecision; /* %.* 1 precision from variable parameter ;0 not */ +} SecFormatAttr; + +typedef union { + char *str; /* not a null terminated string */ +#if SECUREC_HAVE_WCHART + wchar_t *wStr; +#endif +} SecFormatBuf; + +typedef union { + char str[SECUREC_BUFFER_SIZE + 1]; +#ifdef SECUREC_FOR_WCHAR + wchar_t wStr[SECUREC_BUFFER_SIZE + 1]; +#endif +} SecBuffer; + +#if SECUREC_ENABLE_SPRINTF_FLOAT +/* call system sprintf to format float value */ +static int SecIndirectSprintf(char *strDest, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + SECUREC_MASK_MSVC_CRT_WARNING + ret = vsprintf(strDest, format, argList); + SECUREC_END_MASK_MSVC_CRT_WARNING + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +/* out put long double value to dest */ +static int SecFormatLongDboule(char *strDest,const SecFormatAttr *formatAttr, const char *fmt, long double ldValue) +{ + int fldWidth = ((formatAttr->flags & SECUREC_FLAG_LEFT) ? (-(formatAttr->fldWidth)) : formatAttr->fldWidth); + if (formatAttr->dynWidth && formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, fldWidth, formatAttr->precision, ldValue); + } else if (formatAttr->dynWidth) { + return SecIndirectSprintf(strDest, fmt, fldWidth, ldValue); + } else if (formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, formatAttr->precision, ldValue); + } + return SecIndirectSprintf(strDest, fmt, ldValue); +} +#endif + +/* out put double value to dest */ +static int SecFormatDboule(char *strDest, const SecFormatAttr *formatAttr, const char *fmt, double dValue) +{ + int fldWidth = ((formatAttr->flags & SECUREC_FLAG_LEFT) ? (-(formatAttr->fldWidth)) : formatAttr->fldWidth); + if (formatAttr->dynWidth && formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, fldWidth, formatAttr->precision, dValue); + } else if (formatAttr->dynWidth) { + return SecIndirectSprintf(strDest, fmt, fldWidth, dValue); + } else if (formatAttr->dynPrecision) { + return SecIndirectSprintf(strDest, fmt, formatAttr->precision, dValue); + } + return SecIndirectSprintf(strDest, fmt, dValue); +} +#endif + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +/* to clear e506 warning */ +static int SecIsSameSize(size_t sizeA, size_t sizeB) +{ + return sizeA == sizeB; +} +#endif + +#define SECUREC_SPECIAL_DWORD(val32, numBase) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val32) % (numBase)]; \ +} while (((val32) /= (numBase)) != 0) + +#if defined(SECUREC_USE_SPECIAL_DIV64) || (defined(SECUREC_VXWORKS_VERSION_5_4) && !defined(SECUREC_ON_64BITS)) +/* + * Fast divide by 10 algorithm. + * Calculation divisor multiply 0xcccccccccccccccdULL, resultHi64 >> 3 as quotient + */ +static void SecU64Div10(SecUnsignedInt64 divisor, SecUnsignedInt64 *quotient, SecUnsignedInt32 *remainder) +{ + SecUnsignedInt64 mask = 0xffffffffULL; /* use 0xffffffffULL as 32 bit mask */ + SecUnsignedInt64 magicHi = 0xccccccccULL; /* fast divide 10 magic numbers high 32bit 0xccccccccULL */ + SecUnsignedInt64 magicLow = 0xcccccccdULL; /* fast divide 10 magic numbers low 32bit 0xcccccccdULL */ + SecUnsignedInt64 divisorHi = (SecUnsignedInt64)(SECUREC_SHR_DWORD(divisor)); /* hig 32 bit use */ + SecUnsignedInt64 divisorLow = (SecUnsignedInt64)(divisor & mask); /* low 32 bit mask */ + SecUnsignedInt64 factorHi = divisorHi * magicHi; + SecUnsignedInt64 factorLow1 = divisorHi * magicLow; + SecUnsignedInt64 factorLow2 = divisorLow * magicHi; + SecUnsignedInt64 factorLow3 = divisorLow * magicLow; + SecUnsignedInt64 carry = (factorLow1 & mask) + (factorLow2 & mask) + SECUREC_SHR_DWORD(factorLow3); + SecUnsignedInt64 resultHi64 = factorHi + SECUREC_SHR_DWORD(factorLow1) + \ + SECUREC_SHR_DWORD(factorLow2) + SECUREC_SHR_DWORD(carry); + + *quotient = resultHi64 >> 3; /* fast divide 10 magic numbers 3 */ + *remainder = (SecUnsignedInt32)(divisor - ((*quotient) * 10)); /* quotient mul 10 */ + return; +} +#if defined(SECUREC_VXWORKS_VERSION_5_4) && !defined(SECUREC_ON_64BITS) +/* + * Divide function for VXWORKS + */ +static int SecU64Div32(SecUnsignedInt64 divisor, SecUnsignedInt32 radix, + SecUnsignedInt64 *quotient, SecUnsignedInt32 *remainder) +{ + switch (radix) { + case SECUREC_RADIX_DECIMAL: + SecU64Div10(divisor, quotient, remainder); + break; + case SECUREC_RADIX_HEX: + *quotient = divisor >> SECUREC_OFFSET_DIV_HEX; + *remainder = divisor & 0xfULL; /* mask one hex number by 0xfULL */ + break; + case SECUREC_RADIX_OCTAL: + *quotient = divisor >> SECUREC_OFFSET_DIV_OCTAL; + *remainder = divisor & 0x7ULL; /* mask one hex number by 0x7ULL */ + break; + default: + return -1; + } + return 0; +} +#endif +#endif + +#if defined(SECUREC_USE_SPECIAL_DIV64) +/* The compiler does not provide 64 bit division problems */ +#define SECUREC_SPECIAL_QWORD_BASE10(val64) do { \ + SecUnsignedInt64 quotient = 0; \ + SecUnsignedInt32 digit = 0; \ + SecU64Div10((val64), &(quotient), &(digit)); \ + --formatBuf.str; \ + *(formatBuf.str) = digits[digit]; \ + (val64) = quotient; \ +} while ((val64) != 0) +#else +#define SECUREC_SPECIAL_QWORD_BASE10(val64) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val64) % SECUREC_RADIX_DECIMAL]; \ +} while (((val64) /= SECUREC_RADIX_DECIMAL) != 0) +#endif +#define SECUREC_SPECIAL_QWORD(val64, numBase) do { \ + --formatBuf.str; \ + *(formatBuf.str) = digits[(val64) % (numBase)]; \ +} while (((val64) /= (numBase)) != 0) + + +#define SECUREC_SAFE_WRITE_STR_OPT(src, txtLen, outStream, outChars) do { \ + int ii_; \ + for (ii_ = 0; ii_ < (txtLen); ++ii_) { \ + *((SecChar *)(void *)((outStream)->cur)) = *(SecChar *)(src); \ + (outStream)->cur += sizeof(SecChar); \ + (src) = (src) + 1; \ + } \ + (outStream)->count -= (txtLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (txtLen); \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_WRITE_STR(src, txtLen, outStream, outChars) do { \ + if ((txtLen) < 12) { /* performance optimization for mobile number length 12 */ \ + SECUREC_SAFE_WRITE_STR_OPT((src), (txtLen), (outStream), (outChars)); \ + } else { \ + SecDoMemcpy((outStream)->cur, (src), ((size_t)(unsigned int)(txtLen) * (sizeof(SecChar)))); \ + (outStream)->cur += (size_t)((size_t)(unsigned int)(txtLen) * (sizeof(SecChar))); \ + (outStream)->count -= (txtLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (txtLen); \ + } \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_WRITE_CHAR(c, outStream, outChars) do { \ + *((SecChar *)(void *)((outStream)->cur)) = (SecChar)(c); \ + (outStream)->cur += sizeof(SecChar); \ + (outStream)->count -= (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + 1; \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SAFE_PADDING(padChar, padLen, outStream, outChars) do { \ + int ii_; \ + for (ii_ = 0; ii_ < (padLen); ++ii_) { \ + *((SecChar *)(void *)((outStream)->cur)) = (SecChar)(padChar); \ + (outStream)->cur += sizeof(SecChar); \ + } \ + (outStream)->count -= (padLen) * (int)(sizeof(SecChar)); \ + *(outChars) = *(outChars) + (padLen); \ +} SECUREC_WHILE_ZERO + +/* The count variable can be reduced to 0, and the external function complements the \0 terminator. */ +#define SECUREC_IS_REST_BUF_ENOUGH(stream, needLen) ((int)((stream)->count - \ + (int)(needLen) * (int)(sizeof(SecChar))) >= 0) + +#define SECUREC_FMT_STATE_OFFSET 256 +#ifdef SECUREC_FOR_WCHAR +#define SECUREC_FMT_TYPE(c, fmtTable) ((((unsigned int)(int)(c)) <= (unsigned int)(int)SECUREC_CHAR('~')) ? \ + ((fmtTable)[(unsigned char)(c)]) : 0) +#define SECUREC_DECODE_STATE(c, fmtTable, lastState) (SecFmtState)((((fmtTable)[(SECUREC_FMT_TYPE(c, (fmtTable))) * \ + ((unsigned char)STAT_INVALID + 1) + \ + (unsigned char)(lastState) + \ + SECUREC_FMT_STATE_OFFSET]))) +#else +#define SECUREC_DECODE_STATE(c, fmtTable, lastState) (SecFmtState)(((fmtTable)[((fmtTable)[(unsigned char)(c)]) * \ + ((unsigned char)STAT_INVALID + 1) + \ + (unsigned char)(lastState) + \ + SECUREC_FMT_STATE_OFFSET])) +#endif + +static void SecDecodeFlags(SecChar ch, SecFormatAttr *attr) +{ + switch (ch) { + case SECUREC_CHAR(' '): + attr->flags |= SECUREC_FLAG_SIGN_SPACE; + break; + case SECUREC_CHAR('+'): + attr->flags |= SECUREC_FLAG_SIGN; + break; + case SECUREC_CHAR('-'): + attr->flags |= SECUREC_FLAG_LEFT; + break; + case SECUREC_CHAR('0'): + attr->flags |= SECUREC_FLAG_LEADZERO; /* add zero th the front */ + break; + case SECUREC_CHAR('#'): + attr->flags |= SECUREC_FLAG_ALTERNATE; /* output %x with 0x */ + break; + default: + break; + } + return; +} + + +/* + * Decoded size identifier in format string to Reduce the number of lines of function code + */ +static int SecDecodeSizeI(SecFormatAttr *attr, const SecChar **format) +{ +#ifdef SECUREC_ON_64BITS + attr->flags |= SECUREC_FLAG_I64; /* %I to INT64 */ +#endif + if ((**format == SECUREC_CHAR('6')) && (*((*format) + 1) == SECUREC_CHAR('4'))) { + (*format) += 2; /* add 2 to skip I64 */ + attr->flags |= SECUREC_FLAG_I64; /* %I64 to INT64 */ + } else if ((**format == SECUREC_CHAR('3')) && (*((*format) + 1) == SECUREC_CHAR('2'))) { + (*format) += 2; /* add 2 to skip I32 */ + attr->flags &= ~SECUREC_FLAG_I64; /* %I64 to INT32 */ + } else if ((**format == SECUREC_CHAR('d')) || (**format == SECUREC_CHAR('i')) || + (**format == SECUREC_CHAR('o')) || (**format == SECUREC_CHAR('u')) || + (**format == SECUREC_CHAR('x')) || (**format == SECUREC_CHAR('X'))) { + /* do nothing */ + } else { + /* Compatibility code for "%I" just print I */ + return -1; + } + return 0; +} +/* + * Decoded size identifier in format string + */ +static int SecDecodeSize(SecChar ch, SecFormatAttr *attr, const SecChar **format) +{ + switch (ch) { +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('j'): + attr->flags |= SECUREC_FLAG_INTMAX; + break; +#endif + case SECUREC_CHAR('q'): + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('L'): + attr->flags |= SECUREC_FLAG_LONGLONG | SECUREC_FLAG_LONG_DOUBLE; + break; + case SECUREC_CHAR('l'): + if (**format == SECUREC_CHAR('l')) { + *format = *format + 1; + attr->flags |= SECUREC_FLAG_LONGLONG; /* long long */ + } else { + attr->flags |= SECUREC_FLAG_LONG; /* long int or wchar_t */ + } + break; + case SECUREC_CHAR('t'): + attr->flags |= SECUREC_FLAG_PTRDIFF; + break; +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + case SECUREC_CHAR('z'): + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('Z'): + attr->flags |= SECUREC_FLAG_SIZE; + break; +#endif + case SECUREC_CHAR('I'): + if (SecDecodeSizeI(attr, format) != 0) { + /* Compatibility code for "%I" just print I */ + return -1; + } + break; + case SECUREC_CHAR('h'): + if (**format == SECUREC_CHAR('h')) { + attr->flags |= SECUREC_FLAG_CHAR; /* char */ + } else { + attr->flags |= SECUREC_FLAG_SHORT; /* short int */ + } + break; + case SECUREC_CHAR('w'): + attr->flags |= SECUREC_FLAG_WIDECHAR; /* wide char */ + break; + default: + break; + } + return 0; +} + +/* + * Decoded char type identifier + */ +static int SecDecodeTypeC(SecFormatAttr *attr, unsigned int cValue, SecFormatBuf *formatBuf, SecBuffer *buffer) +{ +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) && !(defined(__hpux)) && !(defined(SECUREC_ON_SOLARIS)) + attr->flags &= ~SECUREC_FLAG_LEADZERO; +#endif + +#ifdef SECUREC_FOR_WCHAR + attr->bufferIsWide = 1; + if (attr->flags & SECUREC_FLAG_SHORT) { +#if SECUREC_HAVE_MBTOWC + /* multibyte character to wide character */ + char tmpChar[2]; /* One character string, length is 2 */ + tmpChar[0] = (char)(cValue & 0x00ff); + tmpChar[1] = '\0'; + + if (mbtowc(buffer->wStr, tmpChar, sizeof(tmpChar)) < 0) { + return -1; + } +#else + return -1; +#endif + } else { + buffer->wStr[0] = (wchar_t)cValue; + } + formatBuf->wStr = buffer->wStr; + return 1; /* only 1 wide character */ +#else /* SECUREC_FOR_WCHAR */ + attr->bufferIsWide = 0; + if (attr->flags & (SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR)) { +#if SECUREC_HAVE_WCTOMB + wchar_t wChar = (wchar_t)cValue; + int textLen; + /* wide character to multibyte character */ + SECUREC_MASK_MSVC_CRT_WARNING + textLen = wctomb(buffer->str, wChar); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (textLen < 0) { + return -1; + } + formatBuf->str = buffer->str; + return textLen; +#else + return -1; +#endif + } else { + /* get multibyte character from argument */ + unsigned short temp; + temp = (unsigned short)cValue; + buffer->str[0] = (char)temp; + formatBuf->str = buffer->str; + return 1; /* only 1 character */ + } +#endif + +} + +/* literal string to print null ptr, define it as array rather than const text area + * is to avoid gcc warning with pointing const text with variable + */ +#if SECUREC_HAVE_WCHART +static wchar_t g_wStrNullString[SECUREC_NULL_STRING_SIZE] = { L'(', L'n', L'u', L'l', L'l', L')', L'\0', L'\0' }; +#endif +static char g_strNullString[SECUREC_NULL_STRING_SIZE] = "(null)"; + +static int SecDecodeTypeSchar(const SecFormatAttr *attr, SecFormatBuf *formatBuf) +{ + int finalPrecision = (attr->precision == -1) ? SECUREC_INT_MAX : attr->precision; + int textLen; + + if (formatBuf->str == NULL) { /* NULL passed, use special string */ + formatBuf->str = g_strNullString; + } + if (finalPrecision == SECUREC_INT_MAX) { + /* precision NOT assigned */ + /* The strlen performance is high when the string length is greater than 32 */ + textLen = (int)strlen(formatBuf->str); + } else { + /* precision assigned */ + size_t tmpLen; + SECUREC_CALC_STR_LEN(formatBuf->str, (size_t)(unsigned int)finalPrecision, &tmpLen); + textLen = (int)tmpLen; + } + return textLen; +} + +#if SECUREC_HAVE_WCHART +static int SecDecodeTypeSwchar(SecFormatAttr *attr, SecFormatBuf *formatBuf) +{ + int finalPrecision = (attr->precision == -1) ? SECUREC_INT_MAX : attr->precision; + int textLen; + + attr->bufferIsWide = 1; + if (formatBuf->wStr == NULL) { /* NULL passed, use special string */ + formatBuf->wStr = g_wStrNullString; + } + /* textLen in wchar_t */ + SECUREC_CALC_WSTR_LEN(formatBuf->wStr, finalPrecision, &textLen); + + return textLen; +} +#endif + +/* + * Decoded string identifier + */ +static int SecDecodeTypeS(SecFormatAttr *attr, char *argPtr, SecFormatBuf *formatBuf) +{ + int textLen; +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) && (!defined(SECUREC_ON_UNIX)) + attr->flags &= ~SECUREC_FLAG_LEADZERO; +#endif + formatBuf->str = argPtr; +#ifdef SECUREC_FOR_WCHAR +#if defined(SECUREC_COMPATIBLE_LINUX_FORMAT) + if (!(attr->flags & SECUREC_FLAG_LONG)) { + attr->flags |= SECUREC_FLAG_SHORT; + } +#endif + if (attr->flags & SECUREC_FLAG_SHORT) { + /* textLen now contains length in multibyte chars */ + textLen = SecDecodeTypeSchar(attr, formatBuf); + } else { + /* textLen now contains length in wide chars */ + textLen = SecDecodeTypeSwchar(attr, formatBuf); + } +#else /* SECUREC_FOR_WCHAR */ + if (attr->flags & (SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR)) { + /* textLen now contains length in wide chars */ +#if SECUREC_HAVE_WCHART + textLen = SecDecodeTypeSwchar(attr, formatBuf); +#else + textLen = 0; +#endif + } else { + /* textLen now contains length in multibyte chars */ + textLen = SecDecodeTypeSchar(attr, formatBuf); + } +#endif /* SECUREC_FOR_WCHAR */ + return textLen; +} + +/* + * Write one character to dest buffer + */ +static void SecOutputOneChar(SecChar ch, SecPrintfStream *stream, int *counter) +{ + /* normal state, write character */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, 1)) { /* only one char */ + SECUREC_SAFE_WRITE_CHAR(ch, stream, counter); /* char * cast to wchar * */ + } else { +#ifdef SECUREC_FOR_WCHAR + SecWriteCharW(ch, stream, counter); +#else + /* optimize function call to code */ + *counter = -1; + stream->count = -1; +#endif + } +} + +/* + * Check precison in format + */ +static int SecDecodePrecision(SecChar ch, SecFormatAttr *formatAttr) +{ + if (formatAttr->dynPrecision == 0) { + /* add digit to current precision */ + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(formatAttr->precision)) { + return -1; + } + formatAttr->precision = (int)SECUREC_MUL_TEN((unsigned int)formatAttr->precision) + + (unsigned char)(ch - SECUREC_CHAR('0')); + } else { + if (formatAttr->precision < 0) { + formatAttr->precision = -1; + } + if (formatAttr->precision > SECUREC_MAX_WIDTH_LEN) { + return -1; + } + } + return 0; +} + + +/* + * Check width in format + */ +static int SecDecodeWidth(SecChar ch, SecFormatAttr *formatAttr, SecFmtState lastState) +{ + if (formatAttr->dynWidth == 0) { + if (lastState != STAT_WIDTH) { + formatAttr->fldWidth = 0; + } + if (SECUREC_MUL_TEN_ADD_BEYOND_MAX(formatAttr->fldWidth)) { + return -1; + } + formatAttr->fldWidth = (int)SECUREC_MUL_TEN((unsigned int)formatAttr->fldWidth) + + (unsigned char)(ch - SECUREC_CHAR('0')); + } else { + if (formatAttr->fldWidth < 0) { + formatAttr->flags |= SECUREC_FLAG_LEFT; + formatAttr->fldWidth = (-formatAttr->fldWidth); + if (formatAttr->fldWidth > SECUREC_MAX_WIDTH_LEN) { + return -1; + } + } + } + return 0; +} +#ifdef SECUREC_FOR_WCHAR +/* + * Formatting output core functions for wchar version.Called by a function such as vswprintf_s + * argList must not be declare as const + */ +static int SecOutputSW(SecPrintfStream *stream, const wchar_t *cFormat, va_list argList) +#else +/* + * Formatting output core functions for char version.Called by a function such as vsnprintf_s + */ +static int SecOutputS(SecPrintfStream *stream, const char *cFormat, va_list argList) +#endif +{ + const SecChar *format = cFormat; +#if SECUREC_ENABLE_SPRINTF_FLOAT + char *floatBuf = NULL; +#endif + SecFormatBuf formatBuf; + static const char *itoaUpperDigits = "0123456789ABCDEFX"; + static const char *itoaLowerDigits = "0123456789abcdefx"; + const char *digits = itoaUpperDigits; + unsigned int radix = SECUREC_RADIX_DECIMAL; + int charsOut; /* characters written */ + int prefixLen = 0; /* Must be initialized or compiler alerts */ + int padding = 0; + int textLen; /* length of the text */ + int noOutput = 0; /* Must be initialized or compiler alerts */ + SecFmtState state; + SecFmtState lastState; + SecChar prefix[SECUREC_PREFIX_LEN] = { 0 }; + SecChar ch; /* currently read character */ + static const unsigned char stateTable[SECUREC_STATE_TABLE_SIZE] = { + /* type 0: nospecial meanin; + * 1: '%'; + * 2: '.' + * 3: '*' + * 4: '0' + * 5: '1' ... '9' + * 6: ' ', '+', '-', '#' + * 7: 'h', 'l', 'L', 'F', 'w' , 'N','z','q','t','j' + * 8: 'd','o','u','i','x','X','e','f','g' + */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x06, 0x00, 0x06, 0x02, 0x00, + 0x04, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x08, 0x08, 0x08, 0x00, 0x07, 0x00, 0x00, 0x07, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, 0x07, 0x08, 0x07, 0x00, 0x07, 0x00, 0x00, 0x08, + 0x08, 0x07, 0x00, 0x08, 0x07, 0x08, 0x00, 0x07, 0x08, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + /* fill zero for normal char 128 byte for 0x80 - 0xff */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* state 0: normal + * 1: percent + * 2: flag + * 3: width + * 4: dot + * 5: precis + * 6: size + * 7: type + * 8: invalid + */ + 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x00, 0x00, 0x01, 0x00, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x01, 0x00, 0x00, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, 0x03, 0x03, 0x08, 0x05, + 0x08, 0x08, 0x00, 0x00, 0x00, 0x02, 0x02, 0x03, 0x05, 0x05, 0x08, 0x00, 0x00, 0x00, 0x03, 0x03, + 0x03, 0x05, 0x05, 0x08, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x08, 0x08, 0x08, 0x00, 0x00, 0x00, + 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x00, + 0x00 + }; + + SecFormatAttr formatAttr; + SecBuffer buffer; + formatAttr.flags = 0; + formatAttr.bufferIsWide = 0; /* flag for buffer contains wide chars */ + formatAttr.fldWidth = 0; + formatAttr.precision = 0; + formatAttr.dynWidth = 0; + formatAttr.dynPrecision = 0; + charsOut = 0; + textLen = 0; + state = STAT_NORMAL; /* starting state */ + formatBuf.str = NULL; + + /* loop each format character */ + /* remove format != NULL */ + while ((ch = *format) != SECUREC_CHAR('\0') && charsOut >= 0) { + ++format; + lastState = state; + state = SECUREC_DECODE_STATE(ch, stateTable, lastState); + switch (state) { + case STAT_NORMAL: + SecOutputOneChar(ch, stream, &charsOut); + continue; + case STAT_PERCENT: + /* set default values */ + prefixLen = 0; + noOutput = 0; + formatAttr.flags = 0; + formatAttr.fldWidth = 0; + formatAttr.precision = -1; + formatAttr.bufferIsWide = 0; + formatAttr.dynWidth = 0; + formatAttr.dynPrecision = 0; + break; + case STAT_FLAG: + /* set flag based on which flag character */ + SecDecodeFlags(ch, &formatAttr); + break; + case STAT_WIDTH: + /* update width value */ + if (ch == SECUREC_CHAR('*')) { + /* get width */ + formatAttr.fldWidth = (int)va_arg(argList, int); + formatAttr.dynWidth = 1; + } else { + formatAttr.dynWidth = 0; + } + if (SecDecodeWidth(ch, &formatAttr, lastState) != 0) { + return -1; + } + break; + case STAT_DOT: + formatAttr.precision = 0; + break; + case STAT_PRECIS: + /* update precison value */ + if (ch == SECUREC_CHAR('*')) { + /* get precision from arg list */ + formatAttr.precision = (int)va_arg(argList, int); + formatAttr.dynPrecision = 1; + } else { + formatAttr.dynPrecision = 0; + } + if (SecDecodePrecision(ch, &formatAttr) != 0) { + return -1; + } + break; + case STAT_SIZE: + /* read a size specifier, set the formatAttr.flags based on it */ + if (SecDecodeSize(ch, &formatAttr, &format) != 0) { + /* Compatibility code for "%I" just print I */ + SecOutputOneChar(ch, stream, &charsOut); + state = STAT_NORMAL; + continue; + } + break; + case STAT_TYPE: + switch (ch) { + case SECUREC_CHAR('C'): + /* wide char */ + if (!(formatAttr.flags & (SECUREC_FLAG_SHORT | SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR))) { +#ifdef SECUREC_FOR_WCHAR + formatAttr.flags |= SECUREC_FLAG_SHORT; +#else + formatAttr.flags |= SECUREC_FLAG_WIDECHAR; +#endif + } + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('c'): + do { + unsigned int cValue = (unsigned int)va_arg(argList, int); + textLen = SecDecodeTypeC(&formatAttr, cValue, &formatBuf, &buffer); + if (textLen < 0) { + noOutput = 1; + } + } SECUREC_WHILE_ZERO; + break; + case SECUREC_CHAR('S'): /* wide char string */ + if (!(formatAttr.flags & (SECUREC_FLAG_SHORT | SECUREC_FLAG_LONG | SECUREC_FLAG_WIDECHAR))) { +#ifndef SECUREC_FOR_WCHAR + formatAttr.flags |= SECUREC_FLAG_WIDECHAR; +#else + formatAttr.flags |= SECUREC_FLAG_SHORT; +#endif + } + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('s'): + do { + char *argPtr = (char *)va_arg(argList, char *); + textLen = SecDecodeTypeS(&formatAttr, argPtr, &formatBuf); + } SECUREC_WHILE_ZERO; + break; + case SECUREC_CHAR('n'): + /* higher risk disable it */ + return -1; + case SECUREC_CHAR('E'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('F'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('G'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('A'): /* fall-through */ /* FALLTHRU */ + /* convert format char to lower , use Explicit conversion to clean up compilation warning */ + ch = (SecChar)(ch + ((SecChar)(SECUREC_CHAR('a')) - (SECUREC_CHAR('A')))); + /* fall-through */ + /* FALLTHRU */ + case SECUREC_CHAR('e'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('f'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('g'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('a'): +#if SECUREC_ENABLE_SPRINTF_FLOAT + do { + int bufferSize = 0; /* size of formatBuf.str */ + /* floating point conversion */ + formatBuf.str = buffer.str; /* output buffer for float string with default size */ + + /* compute the precision value */ + if (formatAttr.precision < 0) { + formatAttr.precision = SECUREC_FLOAT_DEFAULT_PRECISION; + } else if (formatAttr.precision == 0 && ch == SECUREC_CHAR('g')) { + formatAttr.precision = 1; + } + + /* calc buffer size to store double value + * The maximum length of SECUREC_MAX_WIDTH_LEN is enough + */ + if (formatAttr.flags & SECUREC_FLAG_LONG_DOUBLE) { + if (formatAttr.precision > (SECUREC_MAX_WIDTH_LEN - SECUREC_FLOAT_BUFSIZE_LB)) { + noOutput = 1; + break; + } + /* Long double needs to meet the basic print length */ + bufferSize = SECUREC_FLOAT_BUFSIZE_LB + formatAttr.precision + SECUREC_FLOAT_BUF_EXT; + } else { + if (formatAttr.precision > (SECUREC_MAX_WIDTH_LEN - SECUREC_FLOAT_BUFSIZE)) { + noOutput = 1; + break; + } + /* Double needs to meet the basic print length */ + bufferSize = SECUREC_FLOAT_BUFSIZE + formatAttr.precision + SECUREC_FLOAT_BUF_EXT; + } + if (formatAttr.fldWidth > bufferSize) { + bufferSize = formatAttr.fldWidth + SECUREC_FLOAT_BUF_EXT; + } + + if (bufferSize > SECUREC_BUFFER_SIZE) { + /* the current vlaue of SECUREC_BUFFER_SIZE could NOT store the + * formatted float string + */ + floatBuf = (char *)SECUREC_MALLOC(((size_t)(unsigned int)bufferSize)); + if (floatBuf != NULL) { + formatBuf.str = floatBuf; + } else { + noOutput = 1; + break; + } + } + + do { + /* add following code to call system sprintf API for float number */ + const SecChar *pFloatFmt = format - 2; /* sub 2 to the position before 'f' or 'g' */ + int k; + int fFmtStrLen; + char fFmtBuf[SECUREC_FMT_STR_LEN]; + char *fFmtStr = fFmtBuf; + char *fFmtHeap = NULL; /* to clear warning */ + + while (SECUREC_CHAR('%') != *pFloatFmt) { /* must meet '%' */ + --pFloatFmt; + } + fFmtStrLen = (int)((format - pFloatFmt) + 1); /* with ending terminator */ + if (fFmtStrLen > SECUREC_FMT_STR_LEN) { + /* if SECUREC_FMT_STR_LEN is NOT enough, alloc a new buffer */ + fFmtHeap = (char *)SECUREC_MALLOC((size_t)((unsigned int)fFmtStrLen)); + if (fFmtHeap == NULL) { + noOutput = 1; + break; + } else { + for (k = 0; k < fFmtStrLen - 1; ++k) { + /* convert wchar to char */ + fFmtHeap[k] = (char)(pFloatFmt[k]); /* copy the format string */ + } + fFmtHeap[k] = '\0'; + + fFmtStr = fFmtHeap; + } + } else { + /* purpose of the repeat code is to solve the tool alarm Redundant_Null_Check */ + for (k = 0; k < fFmtStrLen - 1; ++k) { + /* convert wchar to char */ + fFmtBuf[k] = (char)(pFloatFmt[k]); /* copy the format string */ + } + fFmtBuf[k] = '\0'; + } + + if (formatAttr.flags & SECUREC_FLAG_LONG_DOUBLE) { +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + long double tmp = (long double)va_arg(argList, long double); + textLen = SecFormatLongDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); +#else + double tmp = (double)va_arg(argList, double); + textLen = SecFormatDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); +#endif + } else { + double tmp = (double)va_arg(argList, double); + textLen = SecFormatDboule(formatBuf.str, &formatAttr, fFmtStr, tmp); + } + + if (fFmtHeap != NULL) { + /* if buffer is alloced on heap, free it */ + SECUREC_FREE(fFmtHeap); + fFmtHeap = NULL; + /* to clear e438 last value assigned not used , the compiler will + * optimize this code + */ + (void)fFmtHeap; + } + if (textLen < 0 || textLen >= bufferSize) { + /* bufferSize is large enough, just validation the return value */ + noOutput = 1; + break; + } + + /* no padding ,this variable to calculate amount of padding */ + formatAttr.fldWidth = textLen; + prefixLen = 0; /* no padding ,this variable to calculate amount of padding */ + formatAttr.flags = 0; /* clear all internal formatAttr.flags */ + break; + } SECUREC_WHILE_ZERO; + } SECUREC_WHILE_ZERO; + break; +#else + return -1; +#endif + case SECUREC_CHAR('p'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('X'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('x'): + /* unsigned lower hex output */ + digits = itoaLowerDigits; + radix = SECUREC_RADIX_HEX; + switch (ch) { + case SECUREC_CHAR('p'): + /* print a pointer */ +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; +#else + formatAttr.flags |= SECUREC_FLAG_POINTER; +#endif +#ifdef SECUREC_ON_64BITS + formatAttr.flags |= SECUREC_FLAG_I64; /* converting an int64 */ +#else + formatAttr.flags |= SECUREC_FLAG_LONG; /* converting a long */ +#endif + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) && (!defined(SECUREC_ON_UNIX)) +#if defined(SECUREC_VXWORKS_PLATFORM) + formatAttr.precision = 1; +#else + formatAttr.precision = 0; +#endif + formatAttr.flags |= SECUREC_FLAG_ALTERNATE; /* "0x" is not default prefix in UNIX */ + break; +#else + /* not linux vxwoks */ +#if defined(_AIX) || defined(SECUREC_ON_SOLARIS) + formatAttr.precision = 1; +#else + formatAttr.precision = 2 * sizeof(void *); /* 2 precision of different systems */ +#endif +#endif + +#if defined(SECUREC_ON_UNIX) + break; +#endif + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('X'): /* fall-through */ /* FALLTHRU */ + /* unsigned upper hex output */ + digits = itoaUpperDigits; + break; + default: + break; + } + + if (formatAttr.flags & SECUREC_FLAG_ALTERNATE) { + /* alternate form means '0x' prefix */ + prefix[0] = SECUREC_CHAR('0'); + prefix[1] = (SecChar)(digits[16]); /* 16 for 'x' or 'X' */ + +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT) || defined(SECUREC_VXWORKS_PLATFORM)) + if (ch == 'p') { + prefix[1] = SECUREC_CHAR('x'); + } +#endif +#if defined(_AIX) || defined(SECUREC_ON_SOLARIS) + if (ch == 'p') { + prefixLen = 0; + } else { + prefixLen = SECUREC_PREFIX_LEN; + } +#else + prefixLen = SECUREC_PREFIX_LEN; +#endif + + } + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('o'): /* fall-through */ /* FALLTHRU */ + switch (ch) { + case SECUREC_CHAR('i'): /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('d'): /* fall-through */ /* FALLTHRU */ + /* signed decimal output */ + formatAttr.flags |= SECUREC_FLAG_SIGNED; + /* fall-through */ /* FALLTHRU */ + case SECUREC_CHAR('u'): + radix = SECUREC_RADIX_DECIMAL; + break; + case SECUREC_CHAR('o'): + /* unsigned octal output */ + radix = SECUREC_RADIX_OCTAL; + if (formatAttr.flags & SECUREC_FLAG_ALTERNATE) { + /* alternate form means force a leading 0 */ + formatAttr.flags |= SECUREC_FLAG_FORCE_OCTAL; + } + break; + default: + break; + } + + do { + + SecUnsignedInt64 number = 0; /* number to convert */ + SecInt64 l; /* temp long value */ + + /* read argument into variable l */ + if (formatAttr.flags & SECUREC_FLAG_I64) { + l = (SecInt64)va_arg(argList, SecInt64); + } else if (formatAttr.flags & SECUREC_FLAG_LONGLONG) { + l = (SecInt64)va_arg(argList, SecInt64); + } else +#ifdef SECUREC_ON_64BITS + if (formatAttr.flags & SECUREC_FLAG_LONG) { + l = (long)va_arg(argList, long); + } else +#endif /* SECUREC_ON_64BITS */ + if (formatAttr.flags & SECUREC_FLAG_CHAR) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = (char)va_arg(argList, int); /* sign extend */ + if (l >= 128) { /* 128 on some platform, char is always unsigned */ + SecUnsignedInt64 tmpL = (SecUnsignedInt64)l; + unsigned char tmpCh = (unsigned char)(~(tmpL)); + l = tmpCh + 1; + formatAttr.flags |= SECUREC_FLAG_NEGATIVE; + } + } else { + l = (unsigned char)va_arg(argList, int); /* zero-extend */ + } + + } else if (formatAttr.flags & SECUREC_FLAG_SHORT) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = (short)va_arg(argList, int); /* sign extend */ + } else { + l = (unsigned short)va_arg(argList, int); /* zero-extend */ + } + + } +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + else if (formatAttr.flags & SECUREC_FLAG_PTRDIFF) { + l = (ptrdiff_t)va_arg(argList, ptrdiff_t); /* sign extend */ + } else if (formatAttr.flags & SECUREC_FLAG_SIZE) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + /* No suitable macros were found to handle the branch */ + if (SecIsSameSize(sizeof(size_t), sizeof(long))) { + l = va_arg(argList, long); /* sign extend */ + } else if (SecIsSameSize(sizeof(size_t), sizeof(long long))) { + l = va_arg(argList, long long); /* sign extend */ + } else { + l = va_arg(argList, int); /* sign extend */ + } + } else { + l = (SecInt64)(size_t)va_arg(argList, size_t); /* sign extend */ + } + } else if (formatAttr.flags & SECUREC_FLAG_INTMAX) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = va_arg(argList, SecInt64); /* sign extend */ + } else { + /* sign extend */ + l = (SecInt64)(SecUnsignedInt64)va_arg(argList, SecUnsignedInt64); + } + } +#endif + else { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + l = va_arg(argList, int); /* sign extend */ + } else { + l = (unsigned int)va_arg(argList, int); /* zero-extend */ + } + + } + + /* check for negative; copy into number */ + if ((formatAttr.flags & SECUREC_FLAG_SIGNED) && l < 0) { + number = (SecUnsignedInt64)(-l); + formatAttr.flags |= SECUREC_FLAG_NEGATIVE; + } else { + number = (SecUnsignedInt64)l; + } + + if (((formatAttr.flags & SECUREC_FLAG_I64) == 0) && +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT + ((formatAttr.flags & SECUREC_FLAG_INTMAX) == 0) && +#endif +#ifdef SECUREC_ON_64BITS + ((formatAttr.flags & SECUREC_FLAG_PTRDIFF) == 0) && + ((formatAttr.flags & SECUREC_FLAG_SIZE) == 0) && +#if !defined(SECUREC_COMPATIBLE_WIN_FORMAT) /* on window 64 system sizeof long is 32bit */ + ((formatAttr.flags & SECUREC_FLAG_LONG) == 0) && +#endif +#endif + ((formatAttr.flags & SECUREC_FLAG_LONGLONG) == 0)) { + + number &= 0xffffffff; /* use 0xffffffff as 32 bit mask */ + } + + /* check precision value for default */ + if (formatAttr.precision < 0) { + formatAttr.precision = 1; /* default precision */ + } else { +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; +#else + if (!(formatAttr.flags & SECUREC_FLAG_POINTER)) { + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; + } +#endif + if (formatAttr.precision > SECUREC_MAX_PRECISION) { + formatAttr.precision = SECUREC_MAX_PRECISION; + } + } + + /* Check if data is 0; if so, turn off hex prefix, + * 'p' add 0x prefix, otherwise not add prefix + */ + if (number == 0) { +#if !(defined(SECUREC_VXWORKS_PLATFORM) || defined(__hpux)) + prefixLen = 0; +#else + if ((ch == 'p') && (formatAttr.flags & SECUREC_FLAG_ALTERNATE)) { + prefixLen = SECUREC_PREFIX_LEN; + } else { + prefixLen = 0; + } +#endif + } + + /* Convert data to ASCII */ + formatBuf.str = &buffer.str[SECUREC_BUFFER_SIZE]; + + if (number > 0) { +#ifdef SECUREC_ON_64BITS + switch (radix) { + /* the compiler will optimize each one */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_QWORD_BASE10(number); + break; + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_HEX); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_OCTAL); + break; + default: + break; + } +#else /* for 32 bits system */ + if (number <= 0xFFFFFFFFUL) { + /* in most case, the value to be converted is small value */ + SecUnsignedInt32 n32Tmp = (SecUnsignedInt32)number; + switch (radix) { + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_HEX); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_OCTAL); + break; + +#ifdef _AIX + /* the compiler will optimize div 10 */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_DWORD(n32Tmp, SECUREC_RADIX_DECIMAL); + break; +#else + case SECUREC_RADIX_DECIMAL: + do { + /* fast div 10 */ + SecUnsignedInt32 q; + SecUnsignedInt32 r; + do { + *--formatBuf.str = digits[n32Tmp % SECUREC_RADIX_DECIMAL]; + q = (n32Tmp >> 1) + (n32Tmp >> 2); /* fast div magic 2 */ + q = q + (q >> 4); /* fast div magic 4 */ + q = q + (q >> 8); /* fast div magic 8 */ + q = q + (q >> 16); /* fast div magic 16 */ + q = q >> 3; /* fast div magic 3 */ + r = n32Tmp - SECUREC_MUL_TEN(q); + n32Tmp = (r > 9) ? (q + 1) : q; /* fast div magic 9 */ + } while (n32Tmp != 0); + } SECUREC_WHILE_ZERO; + break; +#endif + default: + break; + } /* end switch */ + } else { + /* the value to be converted is greater than 4G */ +#if defined(SECUREC_VXWORKS_VERSION_5_4) + do { + SecUnsignedInt32 digit = 0; /* ascii value of digit */ + SecUnsignedInt64 quotient = 0; + if (SecU64Div32(number,(SecUnsignedInt32)radix, "ient, &digit) != 0) { + noOutput = 1; + break; + } + *--formatBuf.str = digits[digit]; + number = quotient; + } while (number != 0); +#else + switch (radix) { + /* the compiler will optimize div 10 */ + case SECUREC_RADIX_DECIMAL: + SECUREC_SPECIAL_QWORD_BASE10(number); + break; + case SECUREC_RADIX_OCTAL: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_OCTAL); + break; + case SECUREC_RADIX_HEX: + SECUREC_SPECIAL_QWORD(number, SECUREC_RADIX_HEX); + break; + default: + break; + } +#endif + } +#endif + + } + /* compute length of number,.if textLen > 0, then formatBuf.str must be in buffer.str */ + textLen = (int)(size_t)((char *)&buffer.str[SECUREC_BUFFER_SIZE] - formatBuf.str); + if (formatAttr.precision > textLen) { + int ii; + for (ii = 0; ii < formatAttr.precision - textLen; ++ii) { + *--formatBuf.str = '0'; + } + textLen = formatAttr.precision; + } + + /* Force a leading zero if FORCEOCTAL flag set */ + if ((formatAttr.flags & SECUREC_FLAG_FORCE_OCTAL) && + (textLen == 0 || formatBuf.str[0] != '0')) { + *--formatBuf.str = '0'; + ++textLen; /* add a zero */ + } + } SECUREC_WHILE_ZERO; + break; + default: + break; + } + + while (noOutput < 1) { + if (formatAttr.flags & SECUREC_FLAG_SIGNED) { + if (formatAttr.flags & SECUREC_FLAG_NEGATIVE) { + /* prefix is a '-' */ + prefix[0] = SECUREC_CHAR('-'); + prefixLen = 1; + } else if (formatAttr.flags & SECUREC_FLAG_SIGN) { + /* prefix is '+' */ + prefix[0] = SECUREC_CHAR('+'); + prefixLen = 1; + } else if (formatAttr.flags & SECUREC_FLAG_SIGN_SPACE) { + /* prefix is ' ' */ + prefix[0] = SECUREC_CHAR(' '); + prefixLen = 1; + } + } + +#if defined(SECUREC_COMPATIBLE_LINUX_FORMAT) && (!defined(SECUREC_ON_UNIX)) + if ((formatAttr.flags & SECUREC_FLAG_POINTER) && (textLen == 0)) { + formatAttr.flags &= ~SECUREC_FLAG_LEADZERO; + formatBuf.str = &buffer.str[SECUREC_BUFFER_SIZE - 1]; + *formatBuf.str-- = '\0'; + *formatBuf.str-- = ')'; + *formatBuf.str-- = 'l'; + *formatBuf.str-- = 'i'; + *formatBuf.str-- = 'n'; + *formatBuf.str = '('; + textLen = 5; /* length of (nil) is 5 */ + } +#endif + + /* calculate amount of padding */ + padding = (formatAttr.fldWidth - textLen) - prefixLen; + + /* put out the padding, prefix, and text, in the correct order */ + + if (!(formatAttr.flags & (SECUREC_FLAG_LEFT | SECUREC_FLAG_LEADZERO)) && padding > 0) { + /* pad on left with blanks */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR(' '), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR(' '), padding, stream, &charsOut); + } + } + + /* write prefix */ + if (prefixLen > 0) { + SecChar *pPrefix = prefix; + if (SECUREC_IS_REST_BUF_ENOUGH(stream, prefixLen)) { + /* max prefix len is 2, use loop copy */ /* char * cast to wchar * in WCHAR version */ + SECUREC_SAFE_WRITE_STR_OPT(pPrefix, prefixLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(prefix, prefixLen, stream, &charsOut); + } + } + + if ((formatAttr.flags & SECUREC_FLAG_LEADZERO) && !(formatAttr.flags & SECUREC_FLAG_LEFT) + && padding > 0) { + /* write leading zeros */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR('0'), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR('0'), padding, stream, &charsOut); + } + } + + /* write text */ +#ifndef SECUREC_FOR_WCHAR + if (formatAttr.bufferIsWide != 0 && (textLen > 0)) { +#if SECUREC_HAVE_WCTOMB + wchar_t *p = formatBuf.wStr; + int count = textLen; + while (count > 0) { + char tmpBuf[SECUREC_MB_LEN + 1]; + SECUREC_MASK_MSVC_CRT_WARNING + int retVal = wctomb(tmpBuf, *p); + SECUREC_END_MASK_MSVC_CRT_WARNING + if (retVal <= 0) { + charsOut = -1; + break; + } + SECUREC_WRITE_STRING(tmpBuf, retVal, stream, &charsOut); + --count; + ++p; + } +#else + charsOut = -1; + break; +#endif + } else { + if (SECUREC_IS_REST_BUF_ENOUGH(stream, textLen)) { + SECUREC_SAFE_WRITE_STR(formatBuf.str, textLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(formatBuf.str, textLen, stream, &charsOut); + } + } +#else /* SECUREC_FOR_WCHAR */ + if (formatAttr.bufferIsWide == 0 && textLen > 0) { +#if SECUREC_HAVE_MBTOWC + int count = textLen; + char *p = formatBuf.str; + + while (count > 0) { + wchar_t wChar = L'\0'; + int retVal = mbtowc(&wChar, p, (size_t)MB_CUR_MAX); + if (retVal <= 0) { + charsOut = -1; + break; + } + SecWriteCharW(wChar, stream, &charsOut); + p += retVal; + count -= retVal; + } +#else + charsOut = -1; + break; +#endif + } else { + if (SECUREC_IS_REST_BUF_ENOUGH(stream, textLen)) { + /* char * cast to wchar * */ + SECUREC_SAFE_WRITE_STR(formatBuf.wStr, textLen, stream, &charsOut); + } else { + SECUREC_WRITE_STRING(formatBuf.wStr, textLen, stream, &charsOut); + } + } +#endif /* SECUREC_FOR_WCHAR */ + + if (charsOut >= 0 && (formatAttr.flags & SECUREC_FLAG_LEFT) && padding > 0) { + /* pad on right with blanks */ + if (SECUREC_IS_REST_BUF_ENOUGH(stream, padding)) { + /* char * cast to wchar * */ + SECUREC_SAFE_PADDING(SECUREC_CHAR(' '), padding, stream, &charsOut); + } else { + SECUREC_WRITE_MULTI_CHAR(SECUREC_CHAR(' '), padding, stream, &charsOut); + } + } + break; + } +#if SECUREC_ENABLE_SPRINTF_FLOAT + if (floatBuf != NULL) { + SECUREC_FREE(floatBuf); + floatBuf = NULL; + } +#endif + break; + case STAT_INVALID: + return -1; + default: + return -1; /* input format is wrong, directly return */ + } + } + + if (state != STAT_NORMAL && state != STAT_TYPE) { + return -1; + } + + return charsOut; /* the number of characters written */ +} +#endif /* OUTPUT_INL_2B263E9C_43D8_44BB_B17A_6D2033DECEE5 */ + diff --git a/third_party/securec/src/scanf_s.c b/third_party/securec/src/scanf_s.c new file mode 100644 index 00000000..d2172f7f --- /dev/null +++ b/third_party/securec/src/scanf_s.c @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The scanf_s function is equivalent to fscanf_s with the argument stdin interposed before the arguments to scanf_s + * The scanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ + +int scanf_s(const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vscanf_s(format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/secinput.h b/third_party/securec/src/secinput.h new file mode 100644 index 00000000..806fc7d6 --- /dev/null +++ b/third_party/securec/src/secinput.h @@ -0,0 +1,155 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SEC_INPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#define SEC_INPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#include "securecutil.h" + +#define SECUREC_SCANF_EINVAL (-1) +#define SECUREC_SCANF_ERROR_PARA (-2) + +/* for internal stream flag */ +#define SECUREC_MEM_STR_FLAG 0X01 +#define SECUREC_FILE_STREAM_FLAG 0X02 +#define SECUREC_FROM_STDIN_FLAG 0X04 +#define SECUREC_LOAD_FILE_TO_MEM_FLAG 0X08 + +#define SECUREC_UNINITIALIZED_FILE_POS (-1) +#define SECUREC_BOM_HEADER_SIZE 2 +#define SECUREC_BOM_HEADER_BE_1ST 0xFEU +#define SECUREC_BOM_HEADER_BE_2ST 0xFFU +#define SECUREC_BOM_HEADER_LE_1ST 0xFFU +#define SECUREC_BOM_HEADER_LE_2ST 0xFEU +#define SECUREC_UTF8_BOM_HEADER_SIZE 3 +#define SECUREC_UTF8_BOM_HEADER_1ST 0xEFU +#define SECUREC_UTF8_BOM_HEADER_2ND 0xBBU +#define SECUREC_UTF8_BOM_HEADER_3RD 0xBFU +#define SECUREC_UTF8_LEAD_1ST 0xE0 +#define SECUREC_UTF8_LEAD_2ND 0x80 + +typedef struct { + unsigned int flag; /* mark the properties of input stream */ + int count; /* the size of buffered string in bytes */ + const char *cur; /* the pointer to next read position */ + char *base; /* the pointer to the header of buffered string */ +#if SECUREC_ENABLE_SCANF_FILE + FILE *pf; /* the file pointer */ + long oriFilePos; /* the original position of file offset when fscanf is called */ + int fileRealRead; +#if defined(SECUREC_NO_STD_UNGETC) + unsigned int lastChar; /* the char code of last input */ + int fUnget; /* the boolean flag of pushing a char back to read stream */ +#endif +#endif +} SecFileStream; + + +#define SECUREC_INIT_SEC_FILE_STREAM_COMMON(fileStream, streamFlag, curPtr, strCount) do { \ + (fileStream).flag = (streamFlag); \ + (fileStream).count = (strCount); \ + (fileStream).cur = (curPtr); \ + (fileStream).base = NULL; \ +} SECUREC_WHILE_ZERO + +#if SECUREC_ENABLE_SCANF_FILE +#if defined(SECUREC_NO_STD_UNGETC) +/* This initialization for eliminating redundant initialization. + * Compared with the previous version initialization 0, + * the current code causes the binary size to increase by some bytes + */ +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ + (fileStream).pf = (stream); \ + (fileStream).oriFilePos = (filePos); \ + (fileStream).fileRealRead = 0; \ + (fileStream).lastChar = 0; \ + (fileStream).fUnget = 0; \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ + (fileStream).pf = (stream); \ + (fileStream).oriFilePos = (filePos); \ + (fileStream).fileRealRead = 0; \ +} SECUREC_WHILE_ZERO +#endif +#else /* No SECUREC_ENABLE_SCANF_FILE */ +#define SECUREC_INIT_SEC_FILE_STREAM(fileStream, streamFlag, stream, filePos, curPtr, strCount) do { \ + SECUREC_INIT_SEC_FILE_STREAM_COMMON((fileStream), (streamFlag), (curPtr), (strCount)); \ +} SECUREC_WHILE_ZERO +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + extern int SecInputS(SecFileStream *stream, const char *cFormat, va_list argList); + extern void SecClearDestBuf(const char *buffer, const char *format, va_list argList); +#if SECUREC_IN_KERNEL == 0 + extern int SecInputSW(SecFileStream *stream, const wchar_t *cFormat, va_list argList); + extern void SecClearDestBufW(const wchar_t *buffer, const wchar_t *format, va_list argList); +#endif +/* 20150105 For software and hardware decoupling,such as UMG */ +#if defined(SECUREC_SYSAPI4VXWORKS) +#ifdef feof +#undef feof +#endif + extern int feof(FILE *stream); +#endif + +#if defined(SECUREC_SYSAPI4VXWORKS) || defined(SECUREC_CTYPE_MACRO_ADAPT) +#ifndef isspace +#define isspace(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\r') || ((c) == '\n')) +#endif +#ifndef iswspace +#define iswspace(c) (((c) == L' ') || ((c) == L'\t') || ((c) == L'\r') || ((c) == L'\n')) +#endif +#ifndef isascii +#define isascii(c) (((unsigned char)(c)) <= 0x7f) +#endif +#ifndef isupper +#define isupper(c) ((c) >= 'A' && (c) <= 'Z') +#endif +#ifndef islower +#define islower(c) ((c) >= 'a' && (c) <= 'z') +#endif +#ifndef isalpha +#define isalpha(c) (isupper(c) || (islower(c))) +#endif +#ifndef isdigit +#define isdigit(c) ((c) >= '0' && (c) <= '9') +#endif +#ifndef isxupper +#define isxupper(c) ((c) >= 'A' && (c) <= 'F') +#endif +#ifndef isxlower +#define isxlower(c) ((c) >= 'a' && (c) <= 'f') +#endif +#ifndef isxdigit +#define isxdigit(c) (isdigit(c) || isxupper(c) || isxlower(c)) +#endif +#endif + +#ifdef __cplusplus +} +#endif +/* Reserved file operation macro interface */ +#define SECUREC_LOCK_FILE(s) +#define SECUREC_UNLOCK_FILE(s) +#define SECUREC_LOCK_STDIN(i, s) +#define SECUREC_UNLOCK_STDIN(i, s) +#endif + + diff --git a/third_party/securec/src/securecutil.c b/third_party/securec/src/securecutil.c new file mode 100644 index 00000000..1a44cfbe --- /dev/null +++ b/third_party/securec/src/securecutil.c @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Avoid duplicate header files,not include securecutil.h */ +#include "securecutil.h" + + +#if defined(ANDROID) && (SECUREC_HAVE_WCTOMB || SECUREC_HAVE_MBTOWC) +#include +#if SECUREC_HAVE_WCTOMB +/* + * Convert wide characters to narrow multi-bytes + */ +int wctomb(char *s, wchar_t wc) +{ + return wcrtomb(s, wc, NULL); +} +#endif + +#if SECUREC_HAVE_MBTOWC +/* + * Converting narrow multi-byte characters to wide characters + */ +int mbtowc(wchar_t *pwc, const char *s, size_t n) +{ + return mbrtowc(pwc, s, n, NULL); +} +#endif +#endif + +/* high Num << 8 | num of SPC Ver */ +#define SECUREC_C_VERSION (0x5 << 8) +#define SECUREC_SPC_VERSION 7 +#define SECUREC_VERSION_STR "Huawei Secure C V100R001C01SPC007B002" + +/* SPC verNumber<->verStr like: + * 0X201<->C01 + * 0X202<->SPC001 Redefine numbers after this version + * 0X502<->SPC002 + * 0X503<->SPC003 + * ... + * 0X50a<->SPC010 + * 0X50b<->SPC011 + * ... + */ +/* CP verNumber<->verStr like: + * 0X601<->CP0001 + * 0X602<->CP0002 + * ... + */ +const char *GetHwSecureCVersion(unsigned short *verNumber) +{ + if (verNumber != NULL) { + *verNumber = (unsigned short)(SECUREC_C_VERSION | SECUREC_SPC_VERSION); + } + return SECUREC_VERSION_STR; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(GetHwSecureCVersion); +#endif + diff --git a/third_party/securec/src/securecutil.h b/third_party/securec/src/securecutil.h new file mode 100644 index 00000000..58e6afa5 --- /dev/null +++ b/third_party/securec/src/securecutil.h @@ -0,0 +1,540 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#define SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#include "securec.h" + +#if (defined(_MSC_VER)) && (_MSC_VER >= 1400) +#define SECUREC_MASK_MSVC_CRT_WARNING __pragma(warning(push)) \ + __pragma(warning(disable:4996 4127)) +#define SECUREC_END_MASK_MSVC_CRT_WARNING __pragma(warning(pop)) +#else +#define SECUREC_MASK_MSVC_CRT_WARNING +#define SECUREC_END_MASK_MSVC_CRT_WARNING +#endif +#define SECUREC_WHILE_ZERO SECUREC_MASK_MSVC_CRT_WARNING while (0) SECUREC_END_MASK_MSVC_CRT_WARNING + +#ifndef SECUREC_HAVE_STRNLEN +#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 700) || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200809L) +#if SECUREC_IN_KERNEL +#define SECUREC_HAVE_STRNLEN 0 +#else +#if defined(__GLIBC__) && __GLIBC__ >= 2 && defined(__GLIBC_MINOR__) && __GLIBC_MINOR__ >= 10 +#define SECUREC_HAVE_STRNLEN 1 +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +/* in kernel disbale functions */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 0 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 0 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 0 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 0 +#endif +#else /* no in kernel */ +/* Systems that do not support file, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 1 +#endif +/* Systems that do not support float, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 1 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 1 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 1 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 1 +#endif +#endif + + +#define SECUREC_INT_MAX 2147483647 +#define SECUREC_MUL_SIXTEEN(x) ((x) << 4) +#define SECUREC_MUL_EIGHT(x) ((x) << 3) +#define SECUREC_MUL_TEN(x) ((((x) << 2) + (x)) << 1) +/* Limited format input and output width */ +#define SECUREC_MAX_WIDTH_LEN_DIV_TEN 21474836 +#define SECUREC_MAX_WIDTH_LEN SECUREC_MUL_TEN(SECUREC_MAX_WIDTH_LEN_DIV_TEN) +/* Is the x multiplied by 10 greater than */ +#define SECUREC_MUL_TEN_ADD_BEYOND_MAX(x) (((x) > SECUREC_MAX_WIDTH_LEN_DIV_TEN)) + +#define SECUREC_FLOAT_BUFSIZE (309 + 40) /* Max length of double value */ +#define SECUREC_FLOAT_BUFSIZE_LB (4932 + 40) /* Max length of long double value */ +#define SECUREC_FLOAT_DEFAULT_PRECISION 6 + +/* This macro does not handle pointer equality or integer overflow */ +#define SECUREC_MEMORY_NO_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) <= (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) <= (const char *)(src))) + +#define SECUREC_MEMORY_IS_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) > (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) > (const char *)(src))) + +/* + * Check whether the strings overlap, len is the length of the string not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_NO_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) < (dest)) || \ + ((dest) < (src) && ((dest) + (len)) < (src))) + +/* + * Check whether the strings overlap for strcpy wcscpy function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_IS_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) >= (dest)) || \ + ((dest) < (src) && ((dest) + (len)) >= (src))) + +/* + * Check whether the strings overlap for strcat wcscat function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_CAT_STRING_IS_OVERLAP(dest, destLen, src, srcLen) \ + (((dest) < (src) && ((dest) + (destLen) + (srcLen)) >= (src)) || \ + ((src) < (dest) && ((src) + (srcLen)) >= (dest))) + + +#if SECUREC_HAVE_STRNLEN +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + *(outLen) = strnlen((str), (maxLen)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT(str, maxLen, outLen) do { \ + if ((maxLen) > 8) { \ + /* Optimization or len less then 8 */ \ + if (*((str) + 0) == '\0') { \ + *(outLen) = 0; \ + } else if (*((str) + 1) == '\0') { \ + *(outLen) = 1; \ + } else if (*((str) + 2) == '\0') { \ + *(outLen) = 2; \ + } else if (*((str) + 3) == '\0') { \ + *(outLen) = 3; \ + } else if (*((str) + 4) == '\0') { \ + *(outLen) = 4; \ + } else if (*((str) + 5) == '\0') { \ + *(outLen) = 5; \ + } else if (*((str) + 6) == '\0') { \ + *(outLen) = 6; \ + } else if (*((str) + 7) == '\0') { \ + *(outLen) = 7; \ + } else if (*((str) + 8) == '\0') { \ + /* Optimization with a length of 8 */ \ + *(outLen) = 8; \ + } else { \ + /* The offset is 8 because the performance of 8 byte alignment is high */ \ + *(outLen) = 8 + strnlen((str) + 8, (maxLen) - 8); \ + } \ + } else { \ + SECUREC_CALC_STR_LEN((str), (maxLen), (outLen)); \ + } \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + const char *strEnd = (const char *)(str); \ + size_t availableSize = (size_t)(maxLen); \ + while (availableSize > 0 && *strEnd != '\0') { \ + --availableSize; \ + ++strEnd; \ + } \ + *(outLen) = (size_t)(strEnd - (str)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT SECUREC_CALC_STR_LEN +#endif + +#define SECUREC_CALC_WSTR_LEN(str, maxLen, outLen) do { \ + const wchar_t *strEnd = (const wchar_t *)(str); \ + *(outLen) = 0; \ + while (*(outLen) < (maxLen) && *strEnd != L'\0') { \ + *(outLen) = *(outLen) + 1; \ + ++strEnd; \ + } \ +} SECUREC_WHILE_ZERO + + +#ifdef SECUREC_FORMAT_OUTPUT_INPUT +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) || defined(__ARMCC_VERSION) +typedef __int64 SecInt64; +typedef unsigned __int64 SecUnsignedInt64; +#if defined(__ARMCC_VERSION) +typedef unsigned int SecUnsignedInt32; +#else +typedef unsigned __int32 SecUnsignedInt32; +#endif +#else +typedef unsigned int SecUnsignedInt32; +typedef long long SecInt64; +typedef unsigned long long SecUnsignedInt64; +#endif + +#ifdef SECUREC_FOR_WCHAR +#if defined(SECUREC_VXWORKS_PLATFORM) && !defined(__WINT_TYPE__) +typedef wchar_t wint_t; +#endif +typedef wchar_t SecChar; +typedef wchar_t SecUnsignedChar; +typedef wint_t SecInt; +typedef wint_t SecUnsignedInt; +#else /* no SECUREC_FOR_WCHAR */ +typedef char SecChar; +typedef unsigned char SecUnsignedChar; +typedef int SecInt; +typedef unsigned int SecUnsignedInt; +#endif +#endif + +/* Determine whether the address is 8-byte aligned + * Some systems do not have uintptr_t type, so use NULL to clear tool alarm 507 + */ +#define SECUREC_ADDR_ALIGNED_8(addr) (SecIsAddrAligned8((addr), NULL) == 0) + +/* If you define the memory allocation function, + * you need to define the function prototype. You can define this macro as a header file. + */ +#if defined(SECUREC_MALLOC_PROTOTYPE) +SECUREC_MALLOC_PROTOTYPE +#endif + +#ifndef SECUREC_MALLOC +#define SECUREC_MALLOC(x) malloc((size_t)(x)) +#endif + +#ifndef SECUREC_FREE +#define SECUREC_FREE(x) free((void *)(x)) +#endif + +/* struct for performance */ +typedef struct { + unsigned char buf[1]; /* Performance optimization code structure assignment length 1 bytes */ +} SecStrBuf1; +typedef struct { + unsigned char buf[2]; /* Performance optimization code structure assignment length 2 bytes */ +} SecStrBuf2; +typedef struct { + unsigned char buf[3]; /* Performance optimization code structure assignment length 3 bytes */ +} SecStrBuf3; +typedef struct { + unsigned char buf[4]; /* Performance optimization code structure assignment length 4 bytes */ +} SecStrBuf4; +typedef struct { + unsigned char buf[5]; /* Performance optimization code structure assignment length 5 bytes */ +} SecStrBuf5; +typedef struct { + unsigned char buf[6]; /* Performance optimization code structure assignment length 6 bytes */ +} SecStrBuf6; +typedef struct { + unsigned char buf[7]; /* Performance optimization code structure assignment length 7 bytes */ +} SecStrBuf7; +typedef struct { + unsigned char buf[8]; /* Performance optimization code structure assignment length 8 bytes */ +} SecStrBuf8; +typedef struct { + unsigned char buf[9]; /* Performance optimization code structure assignment length 9 bytes */ +} SecStrBuf9; +typedef struct { + unsigned char buf[10]; /* Performance optimization code structure assignment length 10 bytes */ +} SecStrBuf10; +typedef struct { + unsigned char buf[11]; /* Performance optimization code structure assignment length 11 bytes */ +} SecStrBuf11; +typedef struct { + unsigned char buf[12]; /* Performance optimization code structure assignment length 12 bytes */ +} SecStrBuf12; +typedef struct { + unsigned char buf[13]; /* Performance optimization code structure assignment length 13 bytes */ +} SecStrBuf13; +typedef struct { + unsigned char buf[14]; /* Performance optimization code structure assignment length 14 bytes */ +} SecStrBuf14; +typedef struct { + unsigned char buf[15]; /* Performance optimization code structure assignment length 15 bytes */ +} SecStrBuf15; +typedef struct { + unsigned char buf[16]; /* Performance optimization code structure assignment length 16 bytes */ +} SecStrBuf16; +typedef struct { + unsigned char buf[17]; /* Performance optimization code structure assignment length 17 bytes */ +} SecStrBuf17; +typedef struct { + unsigned char buf[18]; /* Performance optimization code structure assignment length 18 bytes */ +} SecStrBuf18; +typedef struct { + unsigned char buf[19]; /* Performance optimization code structure assignment length 19 bytes */ +} SecStrBuf19; +typedef struct { + unsigned char buf[20]; /* Performance optimization code structure assignment length 20 bytes */ +} SecStrBuf20; +typedef struct { + unsigned char buf[21]; /* Performance optimization code structure assignment length 21 bytes */ +} SecStrBuf21; +typedef struct { + unsigned char buf[22]; /* Performance optimization code structure assignment length 22 bytes */ +} SecStrBuf22; +typedef struct { + unsigned char buf[23]; /* Performance optimization code structure assignment length 23 bytes */ +} SecStrBuf23; +typedef struct { + unsigned char buf[24]; /* Performance optimization code structure assignment length 24 bytes */ +} SecStrBuf24; +typedef struct { + unsigned char buf[25]; /* Performance optimization code structure assignment length 25 bytes */ +} SecStrBuf25; +typedef struct { + unsigned char buf[26]; /* Performance optimization code structure assignment length 26 bytes */ +} SecStrBuf26; +typedef struct { + unsigned char buf[27]; /* Performance optimization code structure assignment length 27 bytes */ +} SecStrBuf27; +typedef struct { + unsigned char buf[28]; /* Performance optimization code structure assignment length 28 bytes */ +} SecStrBuf28; +typedef struct { + unsigned char buf[29]; /* Performance optimization code structure assignment length 29 bytes */ +} SecStrBuf29; +typedef struct { + unsigned char buf[30]; /* Performance optimization code structure assignment length 30 bytes */ +} SecStrBuf30; +typedef struct { + unsigned char buf[31]; /* Performance optimization code structure assignment length 31 bytes */ +} SecStrBuf31; +typedef struct { + unsigned char buf[32]; /* Performance optimization code structure assignment length 32 bytes */ +} SecStrBuf32; +typedef struct { + unsigned char buf[33]; /* Performance optimization code structure assignment length 33 bytes */ +} SecStrBuf33; +typedef struct { + unsigned char buf[34]; /* Performance optimization code structure assignment length 34 bytes */ +} SecStrBuf34; +typedef struct { + unsigned char buf[35]; /* Performance optimization code structure assignment length 35 bytes */ +} SecStrBuf35; +typedef struct { + unsigned char buf[36]; /* Performance optimization code structure assignment length 36 bytes */ +} SecStrBuf36; +typedef struct { + unsigned char buf[37]; /* Performance optimization code structure assignment length 37 bytes */ +} SecStrBuf37; +typedef struct { + unsigned char buf[38]; /* Performance optimization code structure assignment length 38 bytes */ +} SecStrBuf38; +typedef struct { + unsigned char buf[39]; /* Performance optimization code structure assignment length 39 bytes */ +} SecStrBuf39; +typedef struct { + unsigned char buf[40]; /* Performance optimization code structure assignment length 40 bytes */ +} SecStrBuf40; +typedef struct { + unsigned char buf[41]; /* Performance optimization code structure assignment length 41 bytes */ +} SecStrBuf41; +typedef struct { + unsigned char buf[42]; /* Performance optimization code structure assignment length 42 bytes */ +} SecStrBuf42; +typedef struct { + unsigned char buf[43]; /* Performance optimization code structure assignment length 43 bytes */ +} SecStrBuf43; +typedef struct { + unsigned char buf[44]; /* Performance optimization code structure assignment length 44 bytes */ +} SecStrBuf44; +typedef struct { + unsigned char buf[45]; /* Performance optimization code structure assignment length 45 bytes */ +} SecStrBuf45; +typedef struct { + unsigned char buf[46]; /* Performance optimization code structure assignment length 46 bytes */ +} SecStrBuf46; +typedef struct { + unsigned char buf[47]; /* Performance optimization code structure assignment length 47 bytes */ +} SecStrBuf47; +typedef struct { + unsigned char buf[48]; /* Performance optimization code structure assignment length 48 bytes */ +} SecStrBuf48; +typedef struct { + unsigned char buf[49]; /* Performance optimization code structure assignment length 49 bytes */ +} SecStrBuf49; +typedef struct { + unsigned char buf[50]; /* Performance optimization code structure assignment length 50 bytes */ +} SecStrBuf50; +typedef struct { + unsigned char buf[51]; /* Performance optimization code structure assignment length 51 bytes */ +} SecStrBuf51; +typedef struct { + unsigned char buf[52]; /* Performance optimization code structure assignment length 52 bytes */ +} SecStrBuf52; +typedef struct { + unsigned char buf[53]; /* Performance optimization code structure assignment length 53 bytes */ +} SecStrBuf53; +typedef struct { + unsigned char buf[54]; /* Performance optimization code structure assignment length 54 bytes */ +} SecStrBuf54; +typedef struct { + unsigned char buf[55]; /* Performance optimization code structure assignment length 55 bytes */ +} SecStrBuf55; +typedef struct { + unsigned char buf[56]; /* Performance optimization code structure assignment length 56 bytes */ +} SecStrBuf56; +typedef struct { + unsigned char buf[57]; /* Performance optimization code structure assignment length 57 bytes */ +} SecStrBuf57; +typedef struct { + unsigned char buf[58]; /* Performance optimization code structure assignment length 58 bytes */ +} SecStrBuf58; +typedef struct { + unsigned char buf[59]; /* Performance optimization code structure assignment length 59 bytes */ +} SecStrBuf59; +typedef struct { + unsigned char buf[60]; /* Performance optimization code structure assignment length 60 bytes */ +} SecStrBuf60; +typedef struct { + unsigned char buf[61]; /* Performance optimization code structure assignment length 61 bytes */ +} SecStrBuf61; +typedef struct { + unsigned char buf[62]; /* Performance optimization code structure assignment length 62 bytes */ +} SecStrBuf62; +typedef struct { + unsigned char buf[63]; /* Performance optimization code structure assignment length 63 bytes */ +} SecStrBuf63; +typedef struct { + unsigned char buf[64]; /* Performance optimization code structure assignment length 64 bytes */ +} SecStrBuf64; + + + + +/* User can change the error handler by modify the following definition, + * such as logging the detail error in file. + */ +#if defined(_DEBUG) || defined(DEBUG) +#if defined(SECUREC_ERROR_HANDLER_BY_ASSERT) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) assert(msg "invalid argument" == NULL) +#define SECUREC_ERROR_INVALID_RANGE(msg) assert(msg "invalid dest buffer size" == NULL) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) assert(msg "buffer overlap" == NULL) +#elif defined(SECUREC_ERROR_HANDLER_BY_PRINTF) +#if SECUREC_IN_KERNEL +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printk("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printk("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printk("%s buffer overlap\n", msg) +#else +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printf("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printf("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printf("%s buffer overlap\n", msg) +#endif +#elif defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) LogSecureCRuntimeError(msg " EINVAL\n") +#define SECUREC_ERROR_INVALID_RANGE(msg) LogSecureCRuntimeError(msg " ERANGE\n") +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) LogSecureCRuntimeError(msg " EOVERLAP\n") +#else /* no HANDLER is defined */ +#define SECUREC_ERROR_INVALID_PARAMTER(msg) ((void)0) +#define SECUREC_ERROR_INVALID_RANGE(msg) ((void)0) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) ((void)0) +#endif +#else /* no DEBUG */ +#define SECUREC_ERROR_INVALID_PARAMTER(msg) ((void)0) +#define SECUREC_ERROR_INVALID_RANGE(msg) ((void)0) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) ((void)0) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* assembly language memory copy and memory set for X86 or MIPS ... */ +#ifdef SECUREC_USE_ASM + extern void *memcpy_opt(void *, const void *, size_t); + extern void *memset_opt(void *, int, size_t); +#endif + +#if defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) + extern void LogSecureCRuntimeError(const char *errDetail); +#endif + +#ifdef SECUREC_INLINE_DO_MEMCPY +static void SecDoMemcpy(void *dest, const void *src, size_t count) +{ + /* + * if SECUREC_USE_ASM macro is enabled, it will call assembly language function to improve performance. + */ +#ifdef SECUREC_USE_ASM + (void)memcpy_opt(dest, src, count); +#else + /* large enough, let system API do it */ + (void)memcpy(dest, src, count); +#endif +} +#endif + +#ifdef SECUREC_INLINE_DO_MEMSET +static void SecDoMemset(void *dest, int c, size_t count) +{ +#ifdef SECUREC_USE_ASM + (void)memset_opt(dest, c, count); +#else + (void)memset(dest, c, count); +#endif +} +#endif + +#ifdef SECUREC_INLINE_STR_LEN +/* The function compiler will be inlined and not placed in other files */ +static size_t SecStrMinLen(const char *str, size_t maxLen) +{ + size_t len; + SECUREC_CALC_STR_LEN(str, maxLen, &len); + return len; +} +#endif + +#ifdef SECUREC_INLINE_STR_LEN_OPT +/* The function compiler will be inlined and not placed in other files */ +static size_t SecStrMinLenOpt(const char *str, size_t maxLen) +{ + size_t len; + SECUREC_CALC_STR_LEN_OPT(str, maxLen, &len); + return len; +} +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif + diff --git a/third_party/securec/src/secureinput_a.c b/third_party/securec/src/secureinput_a.c new file mode 100644 index 00000000..68ff0ae3 --- /dev/null +++ b/third_party/securec/src/secureinput_a.c @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifdef SECUREC_FOR_WCHAR +#undef SECUREC_FOR_WCHAR +#endif + +#include "secinput.h" + +#include "input.inl" + diff --git a/third_party/securec/src/secureinput_w.c b/third_party/securec/src/secureinput_w.c new file mode 100644 index 00000000..7a4bef42 --- /dev/null +++ b/third_party/securec/src/secureinput_w.c @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* if some platforms don't have wchar.h, dont't include it */ +#if !(defined(SECUREC_VXWORKS_PLATFORM)) +/* This header file is placed below secinput.h, which will cause tool alarm, + * but If there is no macro above, it will cause vs2010 compiling alarm + */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#ifndef __STDC_WANT_SECURE_LIB__ +/* The order of adjustment is to eliminate alarm of Duplicate Block */ +#define __STDC_WANT_SECURE_LIB__ 0 +#endif +#ifndef _CRTIMP_ALTERNATIVE +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif +#endif +#include +#endif +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifndef SECUREC_FOR_WCHAR +#define SECUREC_FOR_WCHAR +#endif + +#include "secinput.h" + +#ifndef WEOF +#define WEOF ((wchar_t)(-1)) +#endif + +#include "input.inl" + diff --git a/third_party/securec/src/secureprintoutput.h b/third_party/securec/src/secureprintoutput.h new file mode 100644 index 00000000..40b48a31 --- /dev/null +++ b/third_party/securec/src/secureprintoutput.h @@ -0,0 +1,97 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SECUREPRINTOUTPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#define SECUREPRINTOUTPUT_H_E950DA2C_902F_4B15_BECD_948E99090D9C +#include "securecutil.h" + +/* flag definitions */ +/* Using macros instead of enumerations is because some of the enumerated types under the compiler are 16bit. */ +#define SECUREC_FLAG_SIGN 0x00001U +#define SECUREC_FLAG_SIGN_SPACE 0x00002U +#define SECUREC_FLAG_LEFT 0x00004U +#define SECUREC_FLAG_LEADZERO 0x00008U +#define SECUREC_FLAG_LONG 0x00010U +#define SECUREC_FLAG_SHORT 0x00020U +#define SECUREC_FLAG_SIGNED 0x00040U +#define SECUREC_FLAG_ALTERNATE 0x00080U +#define SECUREC_FLAG_NEGATIVE 0x00100U +#define SECUREC_FLAG_FORCE_OCTAL 0x00200U +#define SECUREC_FLAG_LONG_DOUBLE 0x00400U +#define SECUREC_FLAG_WIDECHAR 0x00800U +#define SECUREC_FLAG_LONGLONG 0x01000U +#define SECUREC_FLAG_CHAR 0x02000U +#define SECUREC_FLAG_POINTER 0x04000U +#define SECUREC_FLAG_I64 0x08000U +#define SECUREC_FLAG_PTRDIFF 0x10000U +#define SECUREC_FLAG_SIZE 0x20000U +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +#define SECUREC_FLAG_INTMAX 0x40000U +#endif + +/* state definitions. Identify the status of the current format */ +typedef enum { + STAT_NORMAL, + STAT_PERCENT, + STAT_FLAG, + STAT_WIDTH, + STAT_DOT, + STAT_PRECIS, + STAT_SIZE, + STAT_TYPE, + STAT_INVALID +} SecFmtState; + +/* Format output buffer pointer and available size */ +typedef struct { + int count; + char *cur; +} SecPrintfStream; + + +#ifndef SECUREC_BUFFER_SIZE +#ifdef SECUREC_STACK_SIZE_LESS_THAN_1K +/* SECUREC_BUFFER_SIZE Can not be less than 23 , + * the length of the octal representation of 64-bit integers with zero lead + */ +#define SECUREC_BUFFER_SIZE 256 +#else +#define SECUREC_BUFFER_SIZE 512 +#endif +#endif +#if SECUREC_BUFFER_SIZE < 23 +#error SECUREC_BUFFER_SIZE Can not be less than 23 +#endif + +#define SECUREC_MAX_PRECISION SECUREC_BUFFER_SIZE +/* max. # bytes in multibyte char ,see MB_LEN_MAX */ +#define SECUREC_MB_LEN 16 +/* The return value of the internal function, which is returned when truncated */ +#define SECUREC_PRINTF_TRUNCATE (-2) + +#ifdef __cplusplus +extern "C" { +#endif + extern int SecVsnprintfImpl(char *string, size_t count, const char *format, va_list argList); +#if SECUREC_IN_KERNEL == 0 + extern int SecVswprintfImpl(wchar_t *string, size_t sizeInWchar, const wchar_t *format, va_list argList); +#endif +#ifdef __cplusplus +} +#endif + +#endif + + diff --git a/third_party/securec/src/secureprintoutput_a.c b/third_party/securec/src/secureprintoutput_a.c new file mode 100644 index 00000000..c320f21f --- /dev/null +++ b/third_party/securec/src/secureprintoutput_a.c @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifdef SECUREC_FOR_WCHAR +#undef SECUREC_FOR_WCHAR +#endif + +#include "secureprintoutput.h" + +#define SECUREC_CHAR(x) x +#define SECUREC_WRITE_MULTI_CHAR SecWriteMultiChar +#define SECUREC_WRITE_STRING SecWriteString + +#ifndef EOF +#define EOF (-1) +#endif + +/* put a char to output */ +#define SECUREC_PUTC(c, outStream) ((--(outStream)->count >= 0) ? \ + (int)((unsigned int)(unsigned char)(*((outStream)->cur++) = (char)(c)) & 0xff) : EOF) +/* to clear e835 */ +#define SECUREC_PUTC_ZERO(outStream) ((--(outStream)->count >= 0) ? \ + ((*((outStream)->cur++) = (char)('\0'))) : EOF) + +static void SecWriteMultiChar(char ch, int num, SecPrintfStream *f, int *pnumwritten); +static void SecWriteString(const char *string, int len, SecPrintfStream *f, int *pnumwritten); + +#include "output.inl" + +/* + * Wide character formatted output implementation + */ +int SecVsnprintfImpl(char *string, size_t count, const char *format, va_list argList) +{ + SecPrintfStream str; + int retVal; + + str.count = (int)count; /* this count include \0 character, Must be greater than zero */ + str.cur = string; + + retVal = SecOutputS(&str, format, argList); + if ((retVal >= 0) && (SECUREC_PUTC_ZERO(&str) != EOF)) { + return retVal; + } else if (str.count < 0) { + /* the buffer was too small; we return truncation */ + string[count - 1] = '\0'; + return SECUREC_PRINTF_TRUNCATE; + } + string[0] = '\0'; /* empty the dest strDest */ + return -1; +} + +/* + * Sec write Wide character + */ +static void SecWriteMultiChar(char ch, int num, SecPrintfStream *f, int *pnumwritten) +{ + int count = num; + while (count-- > 0) { + if (SECUREC_PUTC(ch, f) == EOF) { + *pnumwritten = -1; + break; + } else { + *pnumwritten = *pnumwritten + 1; + } + } +} + +/* + * Sec write string function + */ +static void SecWriteString(const char *string, int len, SecPrintfStream *f, int *pnumwritten) +{ + const char *str = string; + int count = len; + while (count-- > 0) { + if (SECUREC_PUTC(*str, f) == EOF) { + *pnumwritten = -1; + break; + } else { + *pnumwritten = *pnumwritten + 1; + ++str; + } + } +} + diff --git a/third_party/securec/src/secureprintoutput_w.c b/third_party/securec/src/secureprintoutput_w.c new file mode 100644 index 00000000..9063ab4d --- /dev/null +++ b/third_party/securec/src/secureprintoutput_w.c @@ -0,0 +1,170 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* if some platforms don't have wchar.h, dont't include it */ +#if !(defined(SECUREC_VXWORKS_PLATFORM)) +/* This header file is placed below secinput.h, which will cause tool alarm, + * but if there is no macro above, it will cause compiling alarm + */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +#ifndef _CRTIMP_ALTERNATIVE +#define _CRTIMP_ALTERNATIVE /* comment microsoft *_s function */ +#endif +#ifndef __STDC_WANT_SECURE_LIB__ +#define __STDC_WANT_SECURE_LIB__ 0 +#endif +#endif +#include +#endif + +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#define SECUREC_INLINE_DO_MEMCPY 1 +#define SECUREC_FORMAT_OUTPUT_INPUT 1 +#ifndef SECUREC_FOR_WCHAR +#define SECUREC_FOR_WCHAR +#endif + +#include "secureprintoutput.h" + +#ifndef WEOF +#define WEOF ((wchar_t)(-1)) +#endif + +#define SECUREC_CHAR(x) L ## x +#define SECUREC_WRITE_MULTI_CHAR SecWriteMultiCharW +#define SECUREC_WRITE_STRING SecWriteStringW + +static void SecWriteCharW(wchar_t ch, SecPrintfStream *f, int *pnumwritten); +static void SecWriteMultiCharW(wchar_t ch, int num, SecPrintfStream *f, int *pnumwritten); +static void SecWriteStringW(const wchar_t *string, int len, SecPrintfStream *f, int *pnumwritten); +static int SecPutWcharStrEndingZero(SecPrintfStream *str, int zeroCount); + + +#include "output.inl" + +/* + * Wide character formatted output implementation + */ +int SecVswprintfImpl(wchar_t *string, size_t sizeInWchar, const wchar_t *format, va_list argList) +{ + SecPrintfStream str; + int retVal; /* If initialization causes e838 */ + + str.cur = (char *)string; + /* this count include \0 character, Must be greater than zero */ + str.count = (int)(sizeInWchar * sizeof(wchar_t)); + + retVal = SecOutputSW(&str, format, argList); + if ((retVal >= 0) && SecPutWcharStrEndingZero(&str, (int)sizeof(wchar_t))) { + return (retVal); + } else if (str.count < 0) { + /* the buffer was too small; we return truncation */ + string[sizeInWchar - 1] = L'\0'; + return SECUREC_PRINTF_TRUNCATE; + } + string[0] = L'\0'; + return -1; +} + +/* + * Output one zero character zero into the SecPrintfStream structure + */ +static int SecPutZeroChar(SecPrintfStream *str) +{ + if (str->count > 0) { + *(str->cur) = (char)('\0'); + str->count = str->count - 1; + str->cur = str->cur + 1; + return 0; + } + return -1; +} + +/* + * Output a wide character zero end into the SecPrintfStream structure + */ +static int SecPutWcharStrEndingZero(SecPrintfStream *str, int zeroCount) +{ + int succeed = 0; + int i = 0; + + while (i < zeroCount && (SecPutZeroChar(str) == 0)) { + ++i; + } + if (i == zeroCount) { + succeed = 1; + } + return succeed; +} + + +/* + * Output a wide character into the SecPrintfStream structure + */ +static wchar_t SecPutCharW(wchar_t ch, SecPrintfStream *f) +{ + wchar_t wcRet = 0; + if (((f)->count -= (int)sizeof(wchar_t)) >= 0) { + *(wchar_t *)(void *)(f->cur) = ch; + f->cur += sizeof(wchar_t); + wcRet = ch; + } else { + wcRet = (wchar_t)WEOF; + } + return wcRet; +} + +/* + * Output a wide character into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteCharW(wchar_t ch, SecPrintfStream *f, int *pnumwritten) +{ + if (SecPutCharW(ch, f) == (wchar_t)WEOF) { + *pnumwritten = -1; + } else { + *pnumwritten = *pnumwritten + 1; + } +} + +/* + * Output multiple wide character into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteMultiCharW(wchar_t ch, int num, SecPrintfStream *f, int *pnumwritten) +{ + int count = num; + while (count-- > 0) { + SecWriteCharW(ch, f, pnumwritten); + if (*pnumwritten == -1) { + break; + } + } +} + +/* + * Output a wide string into the SecPrintfStream structure, returns the number of characters written + */ +static void SecWriteStringW(const wchar_t *string, int len, SecPrintfStream *f, int *pnumwritten) +{ + const wchar_t *str = string; + int count = len; + while (count-- > 0) { + SecWriteCharW(*str++, f, pnumwritten); + if (*pnumwritten == -1) { + break; + } + } +} + diff --git a/third_party/securec/src/snprintf_s.c b/third_party/securec/src/snprintf_s.c new file mode 100644 index 00000000..b5253fa3 --- /dev/null +++ b/third_party/securec/src/snprintf_s.c @@ -0,0 +1,112 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +#if SECUREC_ENABLE_SNPRINTF +/* + * + * The snprintf_s function is equivalent to the snprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The snprintf_s function formats and stores count or fewer characters in + * strDest and appends a terminating null. Each argument (if any) is converted + * and output according to the corresponding format specification in format. + * The formatting is consistent with the printf family of functions; If copying + * occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for the output. + * destMax The size of the storage location for output. Size + * in bytes for snprintf_s or size in words for snwprintf_s. + * count Maximum number of character to store. + * format Format-control string. + * ... Optional arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return -1 if count < destMax and the output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + * + */ +int snprintf_s(char *strDest, size_t destMax, size_t count, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsnprintf_s(strDest, destMax, count, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(snprintf_s); +#endif +#endif + +#if SECUREC_SNPRINTF_TRUNCATED +/* + * + * The snprintf_truncated_s function is equivalent to the snprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The snprintf_truncated_s function formats and stores count or fewer characters in + * strDest and appends a terminating null. Each argument (if any) is converted + * and output according to the corresponding format specification in format. + * The formatting is consistent with the printf family of functions; If copying + * occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for the output. + * destMax The size of the storage location for output. Size + * in bytes for snprintf_truncated_s or size in words for snwprintf_s. + * format Format-control string. + * ... Optional arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return destMax-1 if output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + * + */ +int snprintf_truncated_s(char *strDest, size_t destMax, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsnprintf_truncated_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(snprintf_truncated_s); +#endif + +#endif + + diff --git a/third_party/securec/src/sprintf_s.c b/third_party/securec/src/sprintf_s.c new file mode 100644 index 00000000..a5108fcb --- /dev/null +++ b/third_party/securec/src/sprintf_s.c @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The sprintf_s function is equivalent to the sprintf function + * except for the parameter destMax and the explicit runtime-constraints violation + * The sprintf_s function formats and stores a series of characters and values + * in strDest. Each argument (if any) is converted and output according to + * the corresponding format specification in format. The format consists of + * ordinary characters and has the same form and function as the format argument + * for printf. A null character is appended after the last character written. + * If copying occurs between strings that overlap, the behavior is undefined. + * + * + * strDest Storage location for output. + * destMax Maximum number of characters to store. + * format Format-control string. + * ... Optional arguments + * + * + * strDest is updated + * + * + * return the number of bytes stored in strDest, not counting the terminating null character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int sprintf_s(char *strDest, size_t destMax, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsprintf_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(sprintf_s); +#endif + + diff --git a/third_party/securec/src/sscanf_s.c b/third_party/securec/src/sscanf_s.c new file mode 100644 index 00000000..6946724f --- /dev/null +++ b/third_party/securec/src/sscanf_s.c @@ -0,0 +1,60 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The sscanf_s function is equivalent to fscanf_s, + * except that input is obtained from a string (specified by the argument buffer) rather than from a stream + * The sscanf function reads data from buffer into the location given by each + * argument. Every argument must be a pointer to a variable with a type that + * corresponds to a type specifier in format. The format argument controls the + * interpretation of the input fields and has the same form and function as + * the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... The converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int sscanf_s(const char *buffer, const char *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vsscanf_s(buffer, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(sscanf_s); +#endif + + diff --git a/third_party/securec/src/strcat_s.c b/third_party/securec/src/strcat_s.c new file mode 100644 index 00000000..15eb3160 --- /dev/null +++ b/third_party/securec/src/strcat_s.c @@ -0,0 +1,101 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_STR_LEN_OPT 1 +#define SECUREC_INLINE_DO_MEMCPY 1 +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoStrcat(char *strDest, size_t destMax, const char *strSrc) +{ + size_t destLen = SecStrMinLen(strDest, destMax); + /* Only optimize strSrc, do not apply this function to strDest */ + size_t srcLen = SecStrMinLenOpt(strSrc, destMax - destLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = '\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("strcat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = '\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("strcat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen + 1); /* single character length include \0 */ + return EOK; +} + +/* + * + * The strcat_s function appends a copy of the string pointed to by strSrc (including the terminating null character) + * to the end of the string pointed to by strDest. + * The initial character of strSrc overwrites the terminating null character of strDest. + * strcat_s will return EOVERLAP_AND_RESET if the source and destination strings overlap. + * + * Note that the second parameter is the total size of the buffer, not the + * remaining size. + * + * + * strDest Null-terminated destination string buffer. + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid)or + * (strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN) + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strcat_s(char *strDest, size_t destMax, const char *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strcat_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strcat_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return SecDoStrcat(strDest, destMax, strSrc); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strcat_s); +#endif + diff --git a/third_party/securec/src/strcpy_s.c b/third_party/securec/src/strcpy_s.c new file mode 100644 index 00000000..d5b412ea --- /dev/null +++ b/third_party/securec/src/strcpy_s.c @@ -0,0 +1,350 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +#if SECUREC_IN_KERNEL== 0 +#ifndef SECUREC_STRCOPY_THRESHOLD_SIZE +#define SECUREC_STRCOPY_THRESHOLD_SIZE 32UL +#endif + +/* + * Determine whether the address is 8-byte aligned, use static to increase performance + * return 0 is aligned + */ +static int SecIsAddrAligned8(const void *addr, const void *zeroAddr) +{ + return (int)(((size_t)((const char*)addr - (const char*)zeroAddr)) & 7); /* use 7 to check aligned 8 */ +} + +/* The purpose of converting to void is to clean up the alarm */ +#define SECUREC_SMALL_STR_COPY do { \ + if (SECUREC_ADDR_ALIGNED_8(strDest) && SECUREC_ADDR_ALIGNED_8(strSrc)) { \ + /* use struct assignment */ \ + switch (srcStrLen) { \ + case 1: \ + *(SecStrBuf1 *)(void *)strDest = *(const SecStrBuf1 *)(const void *)strSrc; \ + break; \ + case 2: \ + *(SecStrBuf2 *)(void *)strDest = *(const SecStrBuf2 *)(const void *)strSrc; \ + break; \ + case 3: \ + *(SecStrBuf3 *)(void *)strDest = *(const SecStrBuf3 *)(const void *)strSrc; \ + break; \ + case 4: \ + *(SecStrBuf4 *)(void *)strDest = *(const SecStrBuf4 *)(const void *)strSrc; \ + break; \ + case 5: \ + *(SecStrBuf5 *)(void *)strDest = *(const SecStrBuf5 *)(const void *)strSrc; \ + break; \ + case 6: \ + *(SecStrBuf6 *)(void *)strDest = *(const SecStrBuf6 *)(const void *)strSrc; \ + break; \ + case 7: \ + *(SecStrBuf7 *)(void *)strDest = *(const SecStrBuf7 *)(const void *)strSrc; \ + break; \ + case 8: \ + *(SecStrBuf8 *)(void *)strDest = *(const SecStrBuf8 *)(const void *)strSrc; \ + break; \ + case 9: \ + *(SecStrBuf9 *)(void *)strDest = *(const SecStrBuf9 *)(const void *)strSrc; \ + break; \ + case 10: \ + *(SecStrBuf10 *)(void *)strDest = *(const SecStrBuf10 *)(const void *)strSrc; \ + break; \ + case 11: \ + *(SecStrBuf11 *)(void *)strDest = *(const SecStrBuf11 *)(const void *)strSrc; \ + break; \ + case 12: \ + *(SecStrBuf12 *)(void *)strDest = *(const SecStrBuf12 *)(const void *)strSrc; \ + break; \ + case 13: \ + *(SecStrBuf13 *)(void *)strDest = *(const SecStrBuf13 *)(const void *)strSrc; \ + break; \ + case 14: \ + *(SecStrBuf14 *)(void *)strDest = *(const SecStrBuf14 *)(const void *)strSrc; \ + break; \ + case 15: \ + *(SecStrBuf15 *)(void *)strDest = *(const SecStrBuf15 *)(const void *)strSrc; \ + break; \ + case 16: \ + *(SecStrBuf16 *)(void *)strDest = *(const SecStrBuf16 *)(const void *)strSrc; \ + break; \ + case 17: \ + *(SecStrBuf17 *)(void *)strDest = *(const SecStrBuf17 *)(const void *)strSrc; \ + break; \ + case 18: \ + *(SecStrBuf18 *)(void *)strDest = *(const SecStrBuf18 *)(const void *)strSrc; \ + break; \ + case 19: \ + *(SecStrBuf19 *)(void *)strDest = *(const SecStrBuf19 *)(const void *)strSrc; \ + break; \ + case 20: \ + *(SecStrBuf20 *)(void *)strDest = *(const SecStrBuf20 *)(const void *)strSrc; \ + break; \ + case 21: \ + *(SecStrBuf21 *)(void *)strDest = *(const SecStrBuf21 *)(const void *)strSrc; \ + break; \ + case 22: \ + *(SecStrBuf22 *)(void *)strDest = *(const SecStrBuf22 *)(const void *)strSrc; \ + break; \ + case 23: \ + *(SecStrBuf23 *)(void *)strDest = *(const SecStrBuf23 *)(const void *)strSrc; \ + break; \ + case 24: \ + *(SecStrBuf24 *)(void *)strDest = *(const SecStrBuf24 *)(const void *)strSrc; \ + break; \ + case 25: \ + *(SecStrBuf25 *)(void *)strDest = *(const SecStrBuf25 *)(const void *)strSrc; \ + break; \ + case 26: \ + *(SecStrBuf26 *)(void *)strDest = *(const SecStrBuf26 *)(const void *)strSrc; \ + break; \ + case 27: \ + *(SecStrBuf27 *)(void *)strDest = *(const SecStrBuf27 *)(const void *)strSrc; \ + break; \ + case 28: \ + *(SecStrBuf28 *)(void *)strDest = *(const SecStrBuf28 *)(const void *)strSrc; \ + break; \ + case 29: \ + *(SecStrBuf29 *)(void *)strDest = *(const SecStrBuf29 *)(const void *)strSrc; \ + break; \ + case 30: \ + *(SecStrBuf30 *)(void *)strDest = *(const SecStrBuf30 *)(const void *)strSrc; \ + break; \ + case 31: \ + *(SecStrBuf31 *)(void *)strDest = *(const SecStrBuf31 *)(const void *)strSrc; \ + break; \ + case 32: \ + *(SecStrBuf32 *)(void *)strDest = *(const SecStrBuf32 *)(const void *)strSrc; \ + break; \ + default: \ + break; \ + } /* END switch */ \ + } else { \ + char *tmpStrDest = (char *)strDest; \ + const char *tmpStrSrc = (const char *)strSrc; \ + switch (srcStrLen) { \ + case 32: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(tmpStrDest++) = *(tmpStrSrc++); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + break; \ + } \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Check Src Range + */ +static errno_t CheckSrcRange(char *strDest, size_t destMax, const char *strSrc) +{ + size_t tmpDestMax = destMax; + const char *tmpSrc = strSrc; + /* use destMax as boundary checker and destMax must be greater than zero */ + while (*(tmpSrc) != '\0' && tmpDestMax > 0) { + ++tmpSrc; + --tmpDestMax; + } + if (tmpDestMax == 0) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strcpy_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +/* + * Handling errors + */ +errno_t strcpy_error(char *strDest, size_t destMax, const char *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strcpy_s"); + return ERANGE; + } else if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strcpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return CheckSrcRange(strDest, destMax, strSrc); +} + +/* + * Performance optimization. srcStrLen include '\0' + */ +static void SecDoStrcpyOpt(char *strDest, const char *strSrc, size_t srcStrLen) +{ +#if SECUREC_IN_KERNEL + SecDoMemcpy(strDest, strSrc, srcStrLen); +#else + if (srcStrLen > SECUREC_STRCOPY_THRESHOLD_SIZE) { + SecDoMemcpy(strDest, strSrc, srcStrLen); + } else { + SECUREC_SMALL_STR_COPY; + } +#endif +} + +/* + * + * The strcpy_s function copies the string pointed to strSrc + * (including the terminating null character) into the array pointed to by strDest + * The destination string must be large enough to hold the source string, + * including the terminating null character. strcpy_s will return EOVERLAP_AND_RESET + * if the source and destination strings overlap. + * + * + * strDest Location of destination string buffer + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated. + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strcpy_s(char *strDest, size_t destMax, const char *strSrc) +{ + if ((destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN && strDest != NULL && strSrc != NULL && strDest != strSrc)) { + size_t srcStrLen = SecStrMinLen(strSrc, destMax) + 1; /* len include \0 */ + if (srcStrLen <= destMax) { + /* use mem overlap check include \0 */ + if (SECUREC_MEMORY_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen include '\0' */ + SecDoStrcpyOpt(strDest, strSrc, srcStrLen); + return EOK; + } else { + strDest[0] = '\0'; + SECUREC_ERROR_BUFFER_OVERLAP("strcpy_s"); + return EOVERLAP_AND_RESET; + } + } + } + return strcpy_error(strDest, destMax, strSrc); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strcpy_s); +#endif + diff --git a/third_party/securec/src/strncat_s.c b/third_party/securec/src/strncat_s.c new file mode 100644 index 00000000..e28f6d52 --- /dev/null +++ b/third_party/securec/src/strncat_s.c @@ -0,0 +1,120 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoStrncat(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + size_t destLen = SecStrMinLen(strDest, destMax); + /* The strSrc is no longer optimized. The reason is that when count is small, + * the efficiency of strnlen is higher than that of self realization. + */ + size_t srcLen = SecStrMinLen(strSrc, count); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = '\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("strncat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = '\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen); /* no terminator */ + *(strDest + destLen + srcLen) = '\0'; + return EOK; +} + +/* + * + * The strncat_s function appends not more than n successive characters + * (not including the terminating null character) + * from the array pointed to by strSrc to the end of the string pointed to by strDest + * The strncat_s function try to append the first D characters of strSrc to + * the end of strDest, where D is the lesser of count and the length of strSrc. + * If appending those D characters will fit within strDest (whose size is given + * as destMax) and still leave room for a null terminator, then those characters + * are appended, starting at the original terminating null of strDest, and a + * new terminating null is appended; otherwise, strDest[0] is set to the null + * character. + * + * + * strDest Null-terminated destination string. + * destMax Size of the destination buffer. + * strSrc Null-terminated source string. + * count Number of character to append, or truncate. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid)or + * (strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN) + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strncat_s(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE; + } + + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strncat_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == (size_t)(-1)) { + /* Windows internal functions may pass in -1 when calling this function */ + return SecDoStrncat(strDest, destMax, strSrc, destMax); + } +#endif + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncat_s"); + return ERANGE_AND_RESET; + } + return SecDoStrncat(strDest, destMax, strSrc, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strncat_s); +#endif + diff --git a/third_party/securec/src/strncpy_s.c b/third_party/securec/src/strncpy_s.c new file mode 100644 index 00000000..ab0c45d6 --- /dev/null +++ b/third_party/securec/src/strncpy_s.c @@ -0,0 +1,147 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* [Standardize-exceptions] Use unsafe function: Performance-sensitive + * [reason] Always used in the performance critical path, + * and sufficient input validation is performed before calling + */ +#define SECUREC_INLINE_STR_LEN 1 +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) +#define SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count) \ + (((destMax) > 0 && (destMax) <= SECUREC_STRING_MAX_LEN && (strDest) != NULL && (strSrc) != NULL && \ + ((count) <= SECUREC_STRING_MAX_LEN || (count) == ((size_t)(-1))) && (count) > 0)) +#else +#define SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count) \ + (((destMax) > 0 && (destMax) <= SECUREC_STRING_MAX_LEN && (strDest) != NULL && (strSrc) != NULL && \ + (count) <= SECUREC_STRING_MAX_LEN && (count) > 0)) +#endif + +/* + * Check Src Count Range + */ +static errno_t CheckSrcCountRange(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + size_t tmpDestMax = destMax; + size_t tmpCount = count; + const char *endPos = strSrc; + + /* use destMax and count as boundary checker and destMax must be greater than zero */ + while (*(endPos) != '\0' && tmpDestMax > 0 && tmpCount > 0) { + ++endPos; + --tmpCount; + --tmpDestMax; + } + if (tmpDestMax == 0) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } + return EOK; +} + +/* + * Handling errors, when dest euqal src return EOK + */ +errno_t strncpy_error(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE; + } else if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("strncpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } else if (count > SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; /* clear dest string */ + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } else if (count == 0) { + strDest[0] = '\0'; + return EOK; + } + + return CheckSrcCountRange(strDest, destMax, strSrc, count); +} + +/* + * + * The strncpy_s function copies not more than n successive characters (not including the terminating null character) + * from the array pointed to by strSrc to the array pointed to by strDest. + * + * + * strDest Destination string. + * destMax The size of the destination string, in characters. + * strSrc Source string. + * count Number of characters to be copied. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULL and destMax != 0 and destMax <= SECUREC_STRING_MAX_LEN + * ERANGE destMax is 0 and destMax > SECUREC_STRING_MAX_LEN + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t strncpy_s(char *strDest, size_t destMax, const char *strSrc, size_t count) +{ + if (SECUREC_STRNCPY_PARAM_OK(strDest, destMax, strSrc, count)) { + size_t minCpLen; /* use it to store the maxi length limit */ + if (count < destMax) { + minCpLen = SecStrMinLen(strSrc, count); /* no ending terminator */ + } else { + size_t tmpCount = destMax; +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == ((size_t)(-1))) { + tmpCount = destMax - 1; + } +#endif + minCpLen = SecStrMinLen(strSrc, tmpCount); + if (minCpLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("strncpy_s"); + return ERANGE_AND_RESET; + } + } + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, minCpLen) || strDest == strSrc) { + /* Not overlap */ + SecDoMemcpy(strDest, strSrc, minCpLen); /* copy string without terminator */ + strDest[minCpLen] = '\0'; + return EOK; + } else { + strDest[0] = '\0'; + SECUREC_ERROR_BUFFER_OVERLAP("strncpy_s"); + return EOVERLAP_AND_RESET; + } + } + return strncpy_error(strDest, destMax, strSrc, count); +} + +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strncpy_s); +#endif + diff --git a/third_party/securec/src/strtok_s.c b/third_party/securec/src/strtok_s.c new file mode 100644 index 00000000..9e96805c --- /dev/null +++ b/third_party/securec/src/strtok_s.c @@ -0,0 +1,116 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * Find beginning of token (skip over leading delimiters).Note that + * there is no token if this loop sets string to point to the terminal null. + */ +static char *SecFindBegin(char *strToken, const char *strDelimit) +{ + char *token = strToken; + while (*token != '\0') { + const char *ctl = strDelimit; + while (*ctl != '\0' && *ctl != *token) { + ++ctl; + } + if (*ctl == '\0') { /* don't find any delimiter in string header, break the loop */ + break; + } + ++token; + } + return token; +} + +/* + * Find rest of token + */ +static char *SecFindRest(char *strToken, const char *strDelimit) +{ + /* Find the rest of the token. If it is not the end of the string, + * put a null there. + */ + char *token = strToken; + while (*token != '\0') { + const char *ctl = strDelimit; + while (*ctl != '\0' && *ctl != *token) { + ++ctl; + } + if (*ctl != '\0') { /* find a delimiter */ + *token++ = '\0'; /* set string termintor */ + break; + } + ++token; + } + return token; +} + +/* + * Find the final position pointer + */ +static char *SecUpdateToken(char *strToken, const char *strDelimit, char **context) +{ + /* point to updated position */ + char *token = SecFindRest(strToken, strDelimit); + /* record string position for next search in the context */ + *context = token; + /* Determine if a token has been found. */ + if (token == strToken) { + return NULL; + } + return strToken; +} + +/* + * + * The strtok_s function parses a string into a sequence of strToken, + * replace all characters in strToken string that match to strDelimit set with 0. + * On the first call to strtok_s the string to be parsed should be specified in strToken. + * In each subsequent call that should parse the same string, strToken should be NULL + * + * strToken String containing token or tokens. + * strDelimit Set of delimiter characters. + * context Used to store position information between calls + * to strtok_s + * + * context is updated + * + * On the first call returns the address of the first non \0 character, otherwise NULL is returned. + * In subsequent calls, the strtoken is set to NULL, and the context set is the same as the previous call, + * return NULL if the *context string length is equal 0, otherwise return *context. + */ +char *strtok_s(char *strToken, const char *strDelimit, char **context) +{ + char *orgToken = strToken; + /* validate delimiter and string context */ + if (context == NULL || strDelimit == NULL) { + return NULL; + } + /* valid input string and string pointer from where to search */ + if (orgToken == NULL && (*context) == NULL) { + return NULL; + } + /* If string is null, continue searching from previous string position stored in context */ + if (orgToken == NULL) { + orgToken = *context; + } + orgToken = SecFindBegin(orgToken, strDelimit); + return SecUpdateToken(orgToken, strDelimit, context); +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(strtok_s); +#endif + diff --git a/third_party/securec/src/swprintf_s.c b/third_party/securec/src/swprintf_s.c new file mode 100644 index 00000000..ddd5e10d --- /dev/null +++ b/third_party/securec/src/swprintf_s.c @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The swprintf_s function is the wide-character equivalent of the sprintf_s function + * + * + * strDest Storage location for the output. + * destMax Maximum number of characters to store. + * format Format-control string. + * ... Optional arguments + * + * + * strDest is updated + * + * + * return the number of wide characters stored in strDest, not counting the terminating null wide character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int swprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vswprintf_s(strDest, destMax, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/swscanf_s.c b/third_party/securec/src/swscanf_s.c new file mode 100644 index 00000000..e1dbf202 --- /dev/null +++ b/third_party/securec/src/swscanf_s.c @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * The swscanf_s function is the wide-character equivalent of the sscanf_s function + * The swscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. The format argument controls + * the interpretation of the input fields and has the same form and function + * as the format argument for the scanf function. If copying takes place between + * strings that overlap, the behavior is undefined. + * + * + * buffer Stored data. + * format Format control string, see Format Specifications. + * ... Optional arguments. + * + * + * ... the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; The return value does not include fields that were read but not + * assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int swscanf_s(const wchar_t *buffer, const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vswscanf_s(buffer, format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} + + diff --git a/third_party/securec/src/vfscanf_s.c b/third_party/securec/src/vfscanf_s.c new file mode 100644 index 00000000..f935a63c --- /dev/null +++ b/third_party/securec/src/vfscanf_s.c @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" + +/* + * + * The vfscanf_s function is equivalent to fscanf_s, with the variable argument list replaced by argList + * The vfscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same + * form and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vfscanf_s(FILE *stream, const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + if ((stream == NULL) || (format == NULL)) { + SECUREC_ERROR_INVALID_PARAMTER("vfscanf_s"); + return SECUREC_SCANF_EINVAL; + } + if (stream == stdin) { + return vscanf_s(format, argList); + } + + SECUREC_LOCK_FILE(stream); + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FILE_STREAM_FLAG, stream, SECUREC_UNINITIALIZED_FILE_POS, NULL, 0); + retVal = SecInputS(&fStr, format, argList); + SECUREC_UNLOCK_FILE(stream); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vfscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + return retVal; +} + + diff --git a/third_party/securec/src/vfwscanf_s.c b/third_party/securec/src/vfwscanf_s.c new file mode 100644 index 00000000..7ff2a8d4 --- /dev/null +++ b/third_party/securec/src/vfwscanf_s.c @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" + +/* + * + * The vfwscanf_s function is the wide-character equivalent of the vfscanf_s function + * The vfwscanf_s function reads data from the current position of stream into + * the locations given by argument (if any). Each argument must be a pointer + * to a variable of a type that corresponds to a type specifier in format. + * format controls the interpretation of the input fields and has the same form + * and function as the format argument for scanf. + * + * + * stream Pointer to FILE structure. + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vfwscanf_s(FILE *stream, const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + if ((stream == NULL) || (format == NULL)) { + SECUREC_ERROR_INVALID_PARAMTER("vfwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + if (stream == stdin) { + return vwscanf_s(format, argList); + } + + SECUREC_LOCK_FILE(stream); + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FILE_STREAM_FLAG, stream, SECUREC_UNINITIALIZED_FILE_POS, NULL, 0); + retVal = SecInputSW(&fStr, format, argList); + SECUREC_UNLOCK_FILE(stream); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vfwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vscanf_s.c b/third_party/securec/src/vscanf_s.c new file mode 100644 index 00000000..8383edf6 --- /dev/null +++ b/third_party/securec/src/vscanf_s.c @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" + +/* + * + * The vscanf_s function is equivalent to scanf_s, with the variable argument list replaced by argList, + * The vscanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vscanf_s(const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FROM_STDIN_FLAG, stdin, 0, NULL, 0); + /* + * "va_list" has different definition on different platform, so we can't use argList == NULL + * to determine it's invalid. If you has fixed platform, you can check some fields to validate it, + * such as "argList == NULL" or argList.xxx != NULL or *(size_t *)&argList != 0. + */ + if (format == NULL || fStr.pf == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + SECUREC_LOCK_STDIN(0, fStr.pf); + + retVal = SecInputS(&fStr, format, argList); + + SECUREC_UNLOCK_STDIN(0, fStr.pf); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vsnprintf_s.c b/third_party/securec/src/vsnprintf_s.c new file mode 100644 index 00000000..f679a911 --- /dev/null +++ b/third_party/securec/src/vsnprintf_s.c @@ -0,0 +1,148 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secureprintoutput.h" + +#if SECUREC_ENABLE_VSNPRINTF +/* + * + * The vsnprintf_s function is equivalent to the vsnprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The vsnprintf_s function takes a pointer to an argument list, then formats + * and writes up to count characters of the given data to the memory pointed + * to by strDest and appends a terminating null. + * + * + * strDest Storage location for the output. + * destMax The size of the strDest for output. + * count Maximum number of character to write(not including + * the terminating NULL) + * format Format-control string. + * argList pointer to list of arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return -1 if count < destMax and the output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsnprintf_s(char *strDest, size_t destMax, size_t count, const char *format, va_list argList) +{ + int retVal; + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN || + (count > (SECUREC_STRING_MAX_LEN - 1) && count != (size_t)(-1))) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_s"); + return -1; + } + + if (destMax > count) { + retVal = SecVsnprintfImpl(strDest, count + 1, format, argList); + if (retVal == SECUREC_PRINTF_TRUNCATE) { /* lsd add to keep dest buffer not destroyed 2014.2.18 */ + /* the string has been truncated, return -1 */ + return -1; /* to skip error handler, return strlen(strDest) or -1 */ + } + } else { + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (retVal == SECUREC_PRINTF_TRUNCATE && count == (size_t)(-1)) { + return -1; + } +#endif + } + + if (retVal < 0) { + strDest[0] = '\0'; /* empty the dest strDest */ + + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer too small */ + SECUREC_ERROR_INVALID_RANGE("vsnprintf_s"); + } + + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsnprintf_s); +#endif +#endif + +#if SECUREC_SNPRINTF_TRUNCATED +/* + * + * The vsnprintf_truncated_s function is equivalent to the vsnprintf function + * except for the parameter destMax/count and the explicit runtime-constraints violation + * The vsnprintf_truncated_s function takes a pointer to an argument list, then formats + * and writes up to count characters of the given data to the memory pointed + * to by strDest and appends a terminating null. + * + * + * strDest Storage location for the output. + * destMax The size of the strDest for output. + * the terminating NULL) + * format Format-control string. + * argList pointer to list of arguments. + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null + * return -1 if an error occurs. + * return destMax-1 if output string has been truncated + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsnprintf_truncated_s(char *strDest, size_t destMax, const char *format, va_list argList) +{ + int retVal; + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_truncated_s"); + return -1; + } + + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + if (retVal == SECUREC_PRINTF_TRUNCATE) { + return (int)(destMax - 1); /* to skip error handler, return strlen(strDest) */ + } + strDest[0] = '\0'; /* empty the dest strDest */ + SECUREC_ERROR_INVALID_PARAMTER("vsnprintf_truncated_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsnprintf_truncated_s); +#endif +#endif + + diff --git a/third_party/securec/src/vsprintf_s.c b/third_party/securec/src/vsprintf_s.c new file mode 100644 index 00000000..1933de45 --- /dev/null +++ b/third_party/securec/src/vsprintf_s.c @@ -0,0 +1,72 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secureprintoutput.h" + +/* + * + * The vsprintf_s function is equivalent to the vsprintf function + * except for the parameter destMax and the explicit runtime-constraints violation + * The vsprintf_s function takes a pointer to an argument list, and then formats + * and writes the given data to the memory pointed to by strDest. + * The function differ from the non-secure versions only in that the secure + * versions support positional parameters. + * + * + * strDest Storage location for the output. + * destMax Size of strDest + * format Format specification. + * argList pointer to list of arguments + * + * + * strDest is updated + * + * + * return the number of characters written, not including the terminating null character, + * return -1 if an error occurs. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vsprintf_s(char *strDest, size_t destMax, const char *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > SECUREC_STRING_MAX_LEN) { + if (strDest != NULL && destMax > 0 && destMax <= SECUREC_STRING_MAX_LEN) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vsprintf_s"); + return -1; + } + + retVal = SecVsnprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + strDest[0] = '\0'; + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer is too small */ + SECUREC_ERROR_INVALID_RANGE("vsprintf_s"); + } + SECUREC_ERROR_INVALID_PARAMTER("vsprintf_s"); + return -1; + } + + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsprintf_s); +#endif + + diff --git a/third_party/securec/src/vsscanf_s.c b/third_party/securec/src/vsscanf_s.c new file mode 100644 index 00000000..703c4790 --- /dev/null +++ b/third_party/securec/src/vsscanf_s.c @@ -0,0 +1,87 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" +#if defined(SECUREC_VXWORKS_PLATFORM) && (!defined(SECUREC_SYSAPI4VXWORKS) && !defined(SECUREC_CTYPE_MACRO_ADAPT)) +#include +#endif + +/* + * + * vsscanf_s + * + * + * + * The vsscanf_s function is equivalent to sscanf_s, with the variable argument list replaced by argList + * The vsscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. The format argument controls + * the interpretation of the input fields and has the same form and function + * as the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vsscanf_s(const char *buffer, const char *format, va_list argList) +{ + size_t count; /* If initialization causes e838 */ + int retVal; + SecFileStream fStr; + + /* validation section */ + if (buffer == NULL || format == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } + count = strlen(buffer); + if (count == 0 || count > SECUREC_STRING_MAX_LEN) { + SecClearDestBuf(buffer, format, argList); + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } +#ifdef SECUREC_VXWORKS_PLATFORM + /* + * in vxworks platform when buffer is white string, will set first %s argument tu zero.like following useage: + * " \v\f\t\r\n", "%s", str, strSize + * do not check all character, just first and last character then consider it is white string + */ + if (isspace((int)buffer[0]) && isspace((int)buffer[count - 1])) { + SecClearDestBuf(buffer, format, argList); + } +#endif + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_MEM_STR_FLAG, NULL, 0, buffer, (int)count); + retVal = SecInputS(&fStr, format, argList); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vsscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} +#if SECUREC_IN_KERNEL +EXPORT_SYMBOL(vsscanf_s); +#endif + diff --git a/third_party/securec/src/vswprintf_s.c b/third_party/securec/src/vswprintf_s.c new file mode 100644 index 00000000..140a18d1 --- /dev/null +++ b/third_party/securec/src/vswprintf_s.c @@ -0,0 +1,65 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secureprintoutput.h" + + +/* + * + * The vswprintf_s function is the wide-character equivalent of the vsprintf_s function + * + * + * strDest Storage location for the output. + * destMax Size of strDest + * format Format specification. + * argList pointer to list of arguments + * + * + * strDest is updated + * + * + * return the number of wide characters stored in strDest, not counting the terminating null wide character. + * return -1 if an error occurred. + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +int vswprintf_s(wchar_t *strDest, size_t destMax, const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + + if (format == NULL || strDest == NULL || destMax == 0 || destMax > (SECUREC_WCHAR_STRING_MAX_LEN)) { + if (strDest != NULL && destMax > 0) { + strDest[0] = '\0'; + } + SECUREC_ERROR_INVALID_PARAMTER("vswprintf_s"); + return -1; + } + + retVal = SecVswprintfImpl(strDest, destMax, format, argList); + + if (retVal < 0) { + strDest[0] = '\0'; + if (retVal == SECUREC_PRINTF_TRUNCATE) { + /* Buffer too small */ + SECUREC_ERROR_INVALID_RANGE("vswprintf_s"); + } + SECUREC_ERROR_INVALID_PARAMTER("vswprintf_s"); + return -1; + } + + return retVal; +} + + diff --git a/third_party/securec/src/vswscanf_s.c b/third_party/securec/src/vswscanf_s.c new file mode 100644 index 00000000..f4fb49c8 --- /dev/null +++ b/third_party/securec/src/vswscanf_s.c @@ -0,0 +1,78 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" + +static size_t SecWcslen(const wchar_t *s) +{ + const wchar_t *end = s; + while (*end != L'\0') { + ++end; + } + return ((size_t)((end - s))); +} + +/* + * + * The vswscanf_s function is the wide-character equivalent of the vsscanf_s function + * The vsscanf_s function reads data from buffer into the location given by + * each argument. Every argument must be a pointer to a variable with a type + * that corresponds to a type specifier in format. + * The format argument controls the interpretation of the input fields and + * has the same form and function as the format argument for the scanf function. + * If copying takes place between strings that overlap, the behavior is undefined. + * + * + * buffer Stored data + * format Format control string, see Format Specifications. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Each of these functions returns the number of fields successfully converted + * and assigned; the return value does not include fields that were read but + * not assigned. A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vswscanf_s(const wchar_t *buffer, const wchar_t *format, va_list argList) +{ + size_t count; /* If initialization causes e838 */ + SecFileStream fStr; + int retVal; + + /* validation section */ + if (buffer == NULL || format == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + count = SecWcslen(buffer); + if (count == 0 || count > SECUREC_WCHAR_STRING_MAX_LEN) { + SecClearDestBufW(buffer, format, argList); + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_MEM_STR_FLAG, NULL, 0,\ + (const char *)buffer, (int)count * ((int)sizeof(wchar_t))); + retVal = SecInputSW(&fStr, format, argList); + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vswscanf_s"); + return SECUREC_SCANF_EINVAL; + } + return retVal; +} + + diff --git a/third_party/securec/src/vwscanf_s.c b/third_party/securec/src/vwscanf_s.c new file mode 100644 index 00000000..3dcc6cd6 --- /dev/null +++ b/third_party/securec/src/vwscanf_s.c @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "secinput.h" + +/* + * + * The vwscanf_s function is the wide-character equivalent of the vscanf_s function + * The vwscanf_s function is the wide-character version of vscanf_s. The + * function reads data from the standard input stream stdin and writes the + * data into the location that's given by argument. Each argument must be a + * pointer to a variable of a type that corresponds to a type specifier in + * format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * argList pointer to list of arguments + * + * + * argList the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ +int vwscanf_s(const wchar_t *format, va_list argList) +{ + int retVal; /* If initialization causes e838 */ + SecFileStream fStr; + + SECUREC_INIT_SEC_FILE_STREAM(fStr, SECUREC_FROM_STDIN_FLAG, stdin, 0, NULL, 0); + if (format == NULL || fStr.pf == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("vwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + SECUREC_LOCK_STDIN(0, fStr.pf); + + retVal = SecInputSW(&fStr, format, argList); + + SECUREC_UNLOCK_STDIN(0, fStr.pf); + + if (retVal < 0) { + SECUREC_ERROR_INVALID_PARAMTER("vwscanf_s"); + return SECUREC_SCANF_EINVAL; + } + + return retVal; +} + + diff --git a/third_party/securec/src/wcscat_s.c b/third_party/securec/src/wcscat_s.c new file mode 100644 index 00000000..e1329ad4 --- /dev/null +++ b/third_party/securec/src/wcscat_s.c @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoWcscat(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + size_t destLen; + size_t srcLen; + size_t maxCount; /* Store the maximum available count */ + + /* To calculate the length of a wide character, the parameter must be a wide character */ + SECUREC_CALC_WSTR_LEN(strDest, destMax, &destLen); + maxCount = destMax - destLen; + SECUREC_CALC_WSTR_LEN(strSrc, maxCount, &srcLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = L'\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("wcscat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = L'\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("wcscat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, (srcLen + 1) * sizeof(wchar_t)); /* single character length include \0 */ + return EOK; +} + +/* + * + * The wcscat_s function appends a copy of the wide string pointed to by strSrc +* (including the terminating null wide character) + * to the end of the wide string pointed to by strDest. + * The arguments and return value of wcscat_s are wide-character strings. + * + * The wcscat_s function appends strSrc to strDest and terminates the resulting + * string with a null character. The initial character of strSrc overwrites the + * terminating null character of strDest. wcscat_s will return EOVERLAP_AND_RESET if the + * source and destination strings overlap. + * + * Note that the second parameter is the total size of the buffer, not the + * remaining size. + * + * + * strDest Null-terminated destination string buffer. + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid) or + * (strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN) + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcscat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcscat_s"); + return ERANGE; + } + + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcscat_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + + return SecDoWcscat(strDest, destMax, strSrc); +} + + diff --git a/third_party/securec/src/wcscpy_s.c b/third_party/securec/src/wcscpy_s.c new file mode 100644 index 00000000..b2ca863c --- /dev/null +++ b/third_party/securec/src/wcscpy_s.c @@ -0,0 +1,90 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +static errno_t SecDoWcscpy(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + size_t srcStrLen; + + SECUREC_CALC_WSTR_LEN(strSrc, destMax, &srcStrLen); + if (srcStrLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("wcscpy_s"); + return ERANGE_AND_RESET; + } + if (strDest == strSrc) { + return EOK; + } + + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen include '\0' */ + SecDoMemcpy(strDest, strSrc, (srcStrLen + 1) * sizeof(wchar_t)); /* single character length include \0 */ + return EOK; + } else { + strDest[0] = L'\0'; + SECUREC_ERROR_BUFFER_OVERLAP("wcscpy_s"); + return EOVERLAP_AND_RESET; + } +} + +/* + * + * The wcscpy_s function copies the wide string pointed to by strSrc + * (including theterminating null wide character) into the array pointed to by strDest + + * + * strDest Destination string buffer + * destMax Size of the destination string buffer. + * strSrc Null-terminated source string buffer. + * + * + * strDest is updated. + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET destMax <= length of strSrc and strDest != strSrc + * and strDest != NULL and strSrc != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * and strDest != NULL and strSrc !=NULL and strDest != strSrc + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcscpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcscpy_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcscpy_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + return SecDoWcscpy(strDest, destMax, strSrc); +} + + diff --git a/third_party/securec/src/wcsncat_s.c b/third_party/securec/src/wcsncat_s.c new file mode 100644 index 00000000..facbb236 --- /dev/null +++ b/third_party/securec/src/wcsncat_s.c @@ -0,0 +1,117 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +/* + * Befor this function, the basic parameter checking has been done + */ +static errno_t SecDoWcsncat(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + size_t destLen; + size_t srcLen; + + /* To calculate the length of a wide character, the parameter must be a wide character */ + SECUREC_CALC_WSTR_LEN(strDest, destMax, &destLen); + SECUREC_CALC_WSTR_LEN(strSrc, count, &srcLen); + + if (SECUREC_CAT_STRING_IS_OVERLAP(strDest, destLen, strSrc, srcLen)) { + strDest[0] = L'\0'; + if (strDest + destLen <= strSrc && destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_BUFFER_OVERLAP("wcsncat_s"); + return EOVERLAP_AND_RESET; + } + if (srcLen + destLen >= destMax || strDest == strSrc) { + strDest[0] = L'\0'; + if (destLen == destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + return EINVAL_AND_RESET; + } + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE_AND_RESET; + } + SecDoMemcpy(strDest + destLen, strSrc, srcLen * sizeof(wchar_t)); /* no terminator */ + *(strDest + destLen + srcLen) = L'\0'; + return EOK; +} + +/* + * + * The wcsncat_s function appends not more than n successive wide characters + * (not including the terminating null wide character) + * from the array pointed to by strSrc to the end of the wide string pointed to by strDest. + * + * The wcsncat_s function try to append the first D characters of strSrc to + * the end of strDest, where D is the lesser of count and the length of strSrc. + * If appending those D characters will fit within strDest (whose size is + * given as destMax) and still leave room for a null terminator, then those + * characters are appended, starting at the original terminating null of + * strDest, and a new terminating null is appended; otherwise, strDest[0] is + * set to the null character. + * + * + * strDest Null-terminated destination string. + * destMax Size of the destination buffer. + * strSrc Null-terminated source string. + * count Number of character to append, or truncate. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET (strDest unterminated and all other parameters are valid) or + * (strDest != NULL and strSrc is NULLL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN) + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET strDest have not enough space and all other parameters are valid and not overlap + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcsncat_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncat_s"); + if (strDest != NULL) { + strDest[0] = L'\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_WCHAR_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == ((size_t)-1)) { + /* Windows internal functions may pass in -1 when calling this function */ + return SecDoWcsncat(strDest, destMax, strSrc, destMax); + } +#endif + strDest[0] = L'\0'; + SECUREC_ERROR_INVALID_RANGE("wcsncat_s"); + return ERANGE_AND_RESET; + } + return SecDoWcsncat(strDest, destMax, strSrc, count); +} + + diff --git a/third_party/securec/src/wcsncpy_s.c b/third_party/securec/src/wcsncpy_s.c new file mode 100644 index 00000000..8322af1f --- /dev/null +++ b/third_party/securec/src/wcsncpy_s.c @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define SECUREC_INLINE_DO_MEMCPY 1 + +#include "securecutil.h" + +static errno_t SecDoWcsncpy(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + size_t srcStrLen; + if (count < destMax) { + SECUREC_CALC_WSTR_LEN(strSrc, count, &srcStrLen); + } else { + SECUREC_CALC_WSTR_LEN(strSrc, destMax, &srcStrLen); + } + if (srcStrLen == destMax) { + strDest[0] = '\0'; + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE_AND_RESET; + } + if (strDest == strSrc) { + return EOK; + } + if (SECUREC_STRING_NO_OVERLAP(strDest, strSrc, srcStrLen)) { + /* performance optimization srcStrLen not include '\0' */ + SecDoMemcpy(strDest, strSrc, srcStrLen * sizeof(wchar_t)); + *(strDest + srcStrLen) = L'\0'; + return EOK; + } else { + strDest[0] = L'\0'; + SECUREC_ERROR_BUFFER_OVERLAP("wcsncpy_s"); + return EOVERLAP_AND_RESET; + } +} + +/* + * + * The wcsncpy_s function copies not more than n successive wide characters + * (not including the terminating null wide character) + * from the array pointed to by strSrc to the array pointed to by strDest + * + * + * strDest Destination string. + * destMax The size of the destination string, in characters. + * strSrc Source string. + * count Number of characters to be copied. + * + * + * strDest is updated + * + * + * EOK Success + * EINVAL strDest is NULL and destMax != 0 and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * EINVAL_AND_RESET strDest != NULL and strSrc is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_STRING_MAX_LEN + * ERANGE destMax > SECUREC_WCHAR_STRING_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > SECUREC_WCHAR_STRING_MAX_LEN or + * (destMax <= length of strSrc and destMax <= count and strDest != strSrc + * and strDest != NULL and strSrc != NULL and destMax != 0 and + * destMax <= SECUREC_WCHAR_STRING_MAX_LEN and not overlap) + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and all parameters are valid + * + * + * If there is a runtime-constraint violation, strDest[0] will be set to the '\0' when strDest and destMax valid + */ +errno_t wcsncpy_s(wchar_t *strDest, size_t destMax, const wchar_t *strSrc, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_STRING_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE; + } + if (strDest == NULL || strSrc == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("wcsncpy_s"); + if (strDest != NULL) { + strDest[0] = '\0'; + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > SECUREC_WCHAR_STRING_MAX_LEN) { +#ifdef SECUREC_COMPATIBLE_WIN_FORMAT + if (count == (size_t)(-1)) { + return SecDoWcsncpy(strDest, destMax, strSrc, destMax - 1); + } +#endif + strDest[0] = '\0'; /* clear dest string */ + SECUREC_ERROR_INVALID_RANGE("wcsncpy_s"); + return ERANGE_AND_RESET; + } + + if (count == 0) { + strDest[0] = '\0'; + return EOK; + } + + return SecDoWcsncpy(strDest, destMax, strSrc, count); +} + diff --git a/third_party/securec/src/wcstok_s.c b/third_party/securec/src/wcstok_s.c new file mode 100644 index 00000000..f0ebb6e4 --- /dev/null +++ b/third_party/securec/src/wcstok_s.c @@ -0,0 +1,115 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * FindBegin Wide character postion function + */ +static wchar_t *SecFindBeginW(wchar_t *strToken, const wchar_t *strDelimit) +{ + /* Find beginning of token (skip over leading delimiters). Note that + * there is no token if this loop sets string to point to the terminal null. + */ + wchar_t *token = strToken; + while (*token != L'\0') { + const wchar_t *ctl = strDelimit; + while (*ctl != L'\0' && *ctl != *token) { + ++ctl; + } + if (*ctl == L'\0') { + break; + } + ++token; + } + return token; +} + +/* + * FindBegin rest Wide character postion function + */ +static wchar_t *SecFindRestW(wchar_t *strToken, const wchar_t *strDelimit) +{ + /* Find the end of the token. If it is not the end of the string, + * put a null there. + */ + wchar_t *token = strToken; + while (*token != L'\0') { + const wchar_t *ctl = strDelimit; + while (*ctl != L'\0' && *ctl != *token) { + ++ctl; + } + if (*ctl != L'\0') { + *token++ = L'\0'; + break; + } + ++token; + } + return token; +} + +/* + * Update Token wide character function + */ +static wchar_t *SecUpdateTokenW(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context) +{ + /* point to updated position */ + wchar_t *token = SecFindRestW(strToken, strDelimit); + /* Update the context */ + *context = token; + /* Determine if a token has been found. */ + if (token == strToken) { + return NULL; + } + return strToken; +} + +/* + * + * wcstok_s + * + * + * + * The wcstok_s function is the wide-character equivalent of the strtok_s function + * + * + * strToken String containing token or tokens. + * strDelimit Set of delimiter characters. + * context Used to store position information between calls to + * wcstok_s. + * + * + * context is updated + * + * The wcstok_s function is the wide-character equivalent of the strtok_s function + */ +wchar_t *wcstok_s(wchar_t *strToken, const wchar_t *strDelimit, wchar_t **context) +{ + wchar_t *orgToken = strToken; + /* validation section */ + if (context == NULL || strDelimit == NULL) { + return NULL; + } + if (orgToken == NULL && (*context) == NULL) { + return NULL; + } + /* If string==NULL, continue with previous string */ + if (orgToken == NULL) { + orgToken = *context; + } + orgToken = SecFindBeginW(orgToken, strDelimit); + return SecUpdateTokenW(orgToken, strDelimit, context); +} + diff --git a/third_party/securec/src/wmemcpy_s.c b/third_party/securec/src/wmemcpy_s.c new file mode 100644 index 00000000..907d5b10 --- /dev/null +++ b/third_party/securec/src/wmemcpy_s.c @@ -0,0 +1,67 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securecutil.h" + +/* + * + * The wmemcpy_s function copies n successive wide characters + * from the object pointed to by src into the object pointed to by dest.t. + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Buffer to copy from. + * count Number of characters to copy. + * + * + * dest buffer is uptdated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and count <= destMax + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN and count <= destMax + * ERANGE destMax > SECUREC_WCHAR_MEM_MAX_LEN or destMax is 0 or + * (count > destMax and dest is NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN) + * ERANGE_AND_RESET count > destMax and dest != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and + * count <= destMax destMax != 0 and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * and dest != NULL and src != NULL and dest != src + * + * if an error occured, dest will be filled with 0 when dest and destMax valid . + * If the source and destination overlap, the behavior of wmemcpy_s is undefined. + * Use wmemmove_s to handle overlapping regions. + */ +errno_t wmemcpy_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("wmemcpy_s"); + return ERANGE; + } + if (count > destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wmemcpy_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax * sizeof(wchar_t)); + return ERANGE_AND_RESET; + } + return ERANGE; + } + return memcpy_s(dest, destMax * sizeof(wchar_t), src, count * sizeof(wchar_t)); +} + diff --git a/third_party/securec/src/wmemmove_s.c b/third_party/securec/src/wmemmove_s.c new file mode 100644 index 00000000..9ed4dd67 --- /dev/null +++ b/third_party/securec/src/wmemmove_s.c @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securecutil.h" + +/* + * + * The wmemmove_s function copies n successive wide characters from the object pointed + * to by src into the object pointed to by dest. + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Source object. + * count Number of bytes or character to copy. + * + * + * dest is updated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and count <= destMax + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULLL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN and count <= destMax + * ERANGE destMax > SECUREC_WCHAR_MEM_MAX_LEN or destMax is 0 or + * (count > destMax and dest is NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN) + * ERANGE_AND_RESET count > destMax and dest != NULL and destMax != 0 + * and destMax <= SECUREC_WCHAR_MEM_MAX_LEN + * + * + * If an error occured, dest will be filled with 0 when dest and destMax valid. + * If some regions of the source area and the destination overlap, wmemmove_s + * ensures that the original source bytes in the overlapping region are copied + * before being overwritten + */ +errno_t wmemmove_s(wchar_t *dest, size_t destMax, const wchar_t *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_WCHAR_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_PARAMTER("wmemmove_s"); + return ERANGE; + } + if (count > destMax) { + SECUREC_ERROR_INVALID_PARAMTER("wmemmove_s"); + if (dest != NULL) { + (void)memset(dest, 0, destMax * sizeof(wchar_t)); + return ERANGE_AND_RESET; + } + return ERANGE; + } + return memmove_s(dest, destMax * sizeof(wchar_t), src, count * sizeof(wchar_t)); +} + diff --git a/third_party/securec/src/wscanf_s.c b/third_party/securec/src/wscanf_s.c new file mode 100644 index 00000000..88855847 --- /dev/null +++ b/third_party/securec/src/wscanf_s.c @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "securec.h" + +/* + * + * + * The wscanf_s function is the wide-character equivalent of the scanf_s function + * The wscanf_s function reads data from the standard input stream stdin and + * writes the data into the location that's given by argument. Each argument + * must be a pointer to a variable of a type that corresponds to a type specifier + * in format. If copying occurs between strings that overlap, the behavior is + * undefined. + * + * + * format Format control string. + * ... Optional arguments. + * + * + * ... the converted value stored in user assigned address + * + * + * Returns the number of fields successfully converted and assigned; + * the return value does not include fields that were read but not assigned. + * A return value of 0 indicates that no fields were assigned. + * return -1 if an error occurs. + */ + +int wscanf_s(const wchar_t *format, ...) +{ + int ret; /* If initialization causes e838 */ + va_list argList; + + va_start(argList, format); + ret = vwscanf_s(format, argList); + va_end(argList); + (void)argList; /* to clear e438 last value assigned not used , the compiler will optimize this code */ + + return ret; +} +