1. refactor maaslib to modelscope 2. fix UT error 3. support pipeline which does not register default model Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8988388master
| @@ -4,5 +4,5 @@ rm -rf build | |||||
| # update api rst | # update api rst | ||||
| #rm -rf source/api/ | #rm -rf source/api/ | ||||
| #sphinx-apidoc --module-first -o source/api/ ../maas_lib/ | |||||
| #sphinx-apidoc --module-first -o source/api/ ../modelscope/ | |||||
| make html | make html | ||||
| @@ -1,3 +1,3 @@ | |||||
| yapf -r -i maas_lib/ configs/ tests/ setup.py | |||||
| isort -rc maas_lib/ configs/ tests/ setup.py | |||||
| flake8 maas_lib/ configs/ tests/ setup.py | |||||
| yapf -r -i modelscope/ configs/ tests/ setup.py | |||||
| isort -rc modelscope/ configs/ tests/ setup.py | |||||
| flake8 modelscope/ configs/ tests/ setup.py | |||||
| @@ -1,4 +1,4 @@ | |||||
| Copyright 2022-2023 Alibaba MaaS. All rights reserved. | |||||
| Copyright 2022-2023 Alibaba ModelScope. All rights reserved. | |||||
| Apache License | Apache License | ||||
| Version 2.0, January 2004 | Version 2.0, January 2004 | ||||
| @@ -188,7 +188,7 @@ Copyright 2022-2023 Alibaba MaaS. All rights reserved. | |||||
| same "printed page" as the copyright notice for easier | same "printed page" as the copyright notice for easier | ||||
| identification within third-party archives. | identification within third-party archives. | ||||
| Copyright 2020-2022 Alibaba MaaS. | |||||
| Copyright 2020-2022 Alibaba ModelScope. | |||||
| Licensed under the Apache License, Version 2.0 (the "License"); | Licensed under the Apache License, Version 2.0 (the "License"); | ||||
| you may not use this file except in compliance with the License. | you may not use this file except in compliance with the License. | ||||
| @@ -1 +1 @@ | |||||
| recursive-include maas_lib/configs *.py | |||||
| recursive-include modelscope/configs *.py | |||||
| @@ -1,6 +1,6 @@ | |||||
| # Introduction | # Introduction | ||||
| MaaS library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba. | |||||
| ModelScope library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba. | |||||
| # Design doc | # Design doc | ||||
| @@ -1 +1 @@ | |||||
| This folder will host example configs for each model supported by maas_lib. | |||||
| This folder will host example configs for each model supported by modelscope. | |||||
| @@ -1,34 +0,0 @@ | |||||
| maas\_lib.fileio.format package | |||||
| =============================== | |||||
| .. automodule:: maas_lib.fileio.format | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.fileio.format.base module | |||||
| ----------------------------------- | |||||
| .. automodule:: maas_lib.fileio.format.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.fileio.format.json module | |||||
| ----------------------------------- | |||||
| .. automodule:: maas_lib.fileio.format.json | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.fileio.format.yaml module | |||||
| ----------------------------------- | |||||
| .. automodule:: maas_lib.fileio.format.yaml | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,34 +0,0 @@ | |||||
| maas\_lib.fileio package | |||||
| ======================== | |||||
| .. automodule:: maas_lib.fileio | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| maas_lib.fileio.format | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.fileio.file module | |||||
| ---------------------------- | |||||
| .. automodule:: maas_lib.fileio.file | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.fileio.io module | |||||
| -------------------------- | |||||
| .. automodule:: maas_lib.fileio.io | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,18 +0,0 @@ | |||||
| maas\_lib.models.nlp package | |||||
| ============================ | |||||
| .. automodule:: maas_lib.models.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.models.nlp.sequence\_classification\_model module | |||||
| ----------------------------------------------------------- | |||||
| .. automodule:: maas_lib.models.nlp.sequence_classification_model | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,34 +0,0 @@ | |||||
| maas\_lib.models package | |||||
| ======================== | |||||
| .. automodule:: maas_lib.models | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| maas_lib.models.nlp | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.models.base module | |||||
| ---------------------------- | |||||
| .. automodule:: maas_lib.models.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.models.builder module | |||||
| ------------------------------- | |||||
| .. automodule:: maas_lib.models.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,7 +0,0 @@ | |||||
| maas\_lib.pipelines.audio package | |||||
| ================================= | |||||
| .. automodule:: maas_lib.pipelines.audio | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,18 +0,0 @@ | |||||
| maas\_lib.pipelines.cv package | |||||
| ============================== | |||||
| .. automodule:: maas_lib.pipelines.cv | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.pipelines.cv.image\_matting module | |||||
| -------------------------------------------- | |||||
| .. automodule:: maas_lib.pipelines.cv.image_matting | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,7 +0,0 @@ | |||||
| maas\_lib.pipelines.multi\_modal package | |||||
| ======================================== | |||||
| .. automodule:: maas_lib.pipelines.multi_modal | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,50 +0,0 @@ | |||||
| maas\_lib.preprocessors package | |||||
| =============================== | |||||
| .. automodule:: maas_lib.preprocessors | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.preprocessors.base module | |||||
| ----------------------------------- | |||||
| .. automodule:: maas_lib.preprocessors.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.preprocessors.builder module | |||||
| -------------------------------------- | |||||
| .. automodule:: maas_lib.preprocessors.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.preprocessors.common module | |||||
| ------------------------------------- | |||||
| .. automodule:: maas_lib.preprocessors.common | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.preprocessors.image module | |||||
| ------------------------------------ | |||||
| .. automodule:: maas_lib.preprocessors.image | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.preprocessors.nlp module | |||||
| ---------------------------------- | |||||
| .. automodule:: maas_lib.preprocessors.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,30 +0,0 @@ | |||||
| maas\_lib package | |||||
| ================= | |||||
| .. automodule:: maas_lib | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| maas_lib.fileio | |||||
| maas_lib.models | |||||
| maas_lib.pipelines | |||||
| maas_lib.preprocessors | |||||
| maas_lib.utils | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.version module | |||||
| ------------------------ | |||||
| .. automodule:: maas_lib.version | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,18 +0,0 @@ | |||||
| maas\_lib.trainers.nlp package | |||||
| ============================== | |||||
| .. automodule:: maas_lib.trainers.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.trainers.nlp.sequence\_classification\_trainer module | |||||
| --------------------------------------------------------------- | |||||
| .. automodule:: maas_lib.trainers.nlp.sequence_classification_trainer | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,34 +0,0 @@ | |||||
| maas\_lib.trainers package | |||||
| ========================== | |||||
| .. automodule:: maas_lib.trainers | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| maas_lib.trainers.nlp | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.trainers.base module | |||||
| ------------------------------ | |||||
| .. automodule:: maas_lib.trainers.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.trainers.builder module | |||||
| --------------------------------- | |||||
| .. automodule:: maas_lib.trainers.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,58 +0,0 @@ | |||||
| maas\_lib.utils package | |||||
| ======================= | |||||
| .. automodule:: maas_lib.utils | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| maas\_lib.utils.config module | |||||
| ----------------------------- | |||||
| .. automodule:: maas_lib.utils.config | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.utils.constant module | |||||
| ------------------------------- | |||||
| .. automodule:: maas_lib.utils.constant | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.utils.logger module | |||||
| ----------------------------- | |||||
| .. automodule:: maas_lib.utils.logger | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.utils.pymod module | |||||
| ---------------------------- | |||||
| .. automodule:: maas_lib.utils.pymod | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.utils.registry module | |||||
| ------------------------------- | |||||
| .. automodule:: maas_lib.utils.registry | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| maas\_lib.utils.type\_assert module | |||||
| ----------------------------------- | |||||
| .. automodule:: maas_lib.utils.type_assert | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,34 @@ | |||||
| modelscope.fileio.format package | |||||
| ================================ | |||||
| .. automodule:: modelscope.fileio.format | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.fileio.format.base module | |||||
| ------------------------------------ | |||||
| .. automodule:: modelscope.fileio.format.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.fileio.format.json module | |||||
| ------------------------------------ | |||||
| .. automodule:: modelscope.fileio.format.json | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.fileio.format.yaml module | |||||
| ------------------------------------ | |||||
| .. automodule:: modelscope.fileio.format.yaml | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,34 @@ | |||||
| modelscope.fileio package | |||||
| ========================= | |||||
| .. automodule:: modelscope.fileio | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.fileio.format | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.fileio.file module | |||||
| ----------------------------- | |||||
| .. automodule:: modelscope.fileio.file | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.fileio.io module | |||||
| --------------------------- | |||||
| .. automodule:: modelscope.fileio.io | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,18 @@ | |||||
| modelscope.models.cv.cartoon.facelib.LK package | |||||
| =============================================== | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.LK | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.cv.cartoon.facelib.LK.lk module | |||||
| ------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.LK.lk | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,50 @@ | |||||
| modelscope.models.cv.cartoon.facelib package | |||||
| ============================================ | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.models.cv.cartoon.facelib.LK | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.cv.cartoon.facelib.config module | |||||
| -------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.config | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.cv.cartoon.facelib.face\_detector module | |||||
| ---------------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.face_detector | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.cv.cartoon.facelib.face\_landmark module | |||||
| ---------------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.face_landmark | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.cv.cartoon.facelib.facer module | |||||
| ------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.facelib.facer | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,15 @@ | |||||
| modelscope.models.cv.cartoon.mtcnn\_pytorch package | |||||
| =================================================== | |||||
| .. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.models.cv.cartoon.mtcnn_pytorch.src | |||||
| @@ -0,0 +1,26 @@ | |||||
| modelscope.models.cv.cartoon.mtcnn\_pytorch.src package | |||||
| ======================================================= | |||||
| .. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.cv.cartoon.mtcnn\_pytorch.src.align\_trans module | |||||
| ------------------------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.cv.cartoon.mtcnn\_pytorch.src.matlab\_cp2tform module | |||||
| ----------------------------------------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.matlab_cp2tform | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,27 @@ | |||||
| modelscope.models.cv.cartoon package | |||||
| ==================================== | |||||
| .. automodule:: modelscope.models.cv.cartoon | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.models.cv.cartoon.facelib | |||||
| modelscope.models.cv.cartoon.mtcnn_pytorch | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.cv.cartoon.utils module | |||||
| ----------------------------------------- | |||||
| .. automodule:: modelscope.models.cv.cartoon.utils | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,15 @@ | |||||
| modelscope.models.cv package | |||||
| ============================ | |||||
| .. automodule:: modelscope.models.cv | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.models.cv.cartoon | |||||
| @@ -0,0 +1,26 @@ | |||||
| modelscope.models.nlp package | |||||
| ============================= | |||||
| .. automodule:: modelscope.models.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.nlp.sequence\_classification\_model module | |||||
| ------------------------------------------------------------ | |||||
| .. automodule:: modelscope.models.nlp.sequence_classification_model | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.nlp.text\_generation\_model module | |||||
| ---------------------------------------------------- | |||||
| .. automodule:: modelscope.models.nlp.text_generation_model | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,35 @@ | |||||
| modelscope.models package | |||||
| ========================= | |||||
| .. automodule:: modelscope.models | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.models.cv | |||||
| modelscope.models.nlp | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.models.base module | |||||
| ----------------------------- | |||||
| .. automodule:: modelscope.models.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.models.builder module | |||||
| -------------------------------- | |||||
| .. automodule:: modelscope.models.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,7 @@ | |||||
| modelscope.pipelines.audio package | |||||
| ================================== | |||||
| .. automodule:: modelscope.pipelines.audio | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,26 @@ | |||||
| modelscope.pipelines.cv package | |||||
| =============================== | |||||
| .. automodule:: modelscope.pipelines.cv | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.pipelines.cv.image\_cartoon\_pipeline module | |||||
| ------------------------------------------------------- | |||||
| .. automodule:: modelscope.pipelines.cv.image_cartoon_pipeline | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.pipelines.cv.image\_matting\_pipeline module | |||||
| ------------------------------------------------------- | |||||
| .. automodule:: modelscope.pipelines.cv.image_matting_pipeline | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,18 @@ | |||||
| modelscope.pipelines.multi\_modal package | |||||
| ========================================= | |||||
| .. automodule:: modelscope.pipelines.multi_modal | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.pipelines.multi\_modal.image\_captioning module | |||||
| ---------------------------------------------------------- | |||||
| .. automodule:: modelscope.pipelines.multi_modal.image_captioning | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,26 @@ | |||||
| modelscope.pipelines.nlp package | |||||
| ================================ | |||||
| .. automodule:: modelscope.pipelines.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.pipelines.nlp.sequence\_classification\_pipeline module | |||||
| ------------------------------------------------------------------ | |||||
| .. automodule:: modelscope.pipelines.nlp.sequence_classification_pipeline | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.pipelines.nlp.text\_generation\_pipeline module | |||||
| ---------------------------------------------------------- | |||||
| .. automodule:: modelscope.pipelines.nlp.text_generation_pipeline | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,53 @@ | |||||
| modelscope.pipelines package | |||||
| ============================ | |||||
| .. automodule:: modelscope.pipelines | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.pipelines.audio | |||||
| modelscope.pipelines.cv | |||||
| modelscope.pipelines.multi_modal | |||||
| modelscope.pipelines.nlp | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.pipelines.base module | |||||
| -------------------------------- | |||||
| .. automodule:: modelscope.pipelines.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.pipelines.builder module | |||||
| ----------------------------------- | |||||
| .. automodule:: modelscope.pipelines.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.pipelines.default module | |||||
| ----------------------------------- | |||||
| .. automodule:: modelscope.pipelines.default | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.pipelines.util module | |||||
| -------------------------------- | |||||
| .. automodule:: modelscope.pipelines.util | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,50 @@ | |||||
| modelscope.preprocessors package | |||||
| ================================ | |||||
| .. automodule:: modelscope.preprocessors | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.preprocessors.base module | |||||
| ------------------------------------ | |||||
| .. automodule:: modelscope.preprocessors.base | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.preprocessors.builder module | |||||
| --------------------------------------- | |||||
| .. automodule:: modelscope.preprocessors.builder | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.preprocessors.common module | |||||
| -------------------------------------- | |||||
| .. automodule:: modelscope.preprocessors.common | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.preprocessors.image module | |||||
| ------------------------------------- | |||||
| .. automodule:: modelscope.preprocessors.image | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.preprocessors.nlp module | |||||
| ----------------------------------- | |||||
| .. automodule:: modelscope.preprocessors.nlp | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,18 @@ | |||||
| modelscope.pydatasets package | |||||
| ============================= | |||||
| .. automodule:: modelscope.pydatasets | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.pydatasets.py\_dataset module | |||||
| ---------------------------------------- | |||||
| .. automodule:: modelscope.pydatasets.py_dataset | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -0,0 +1,32 @@ | |||||
| modelscope package | |||||
| ================== | |||||
| .. automodule:: modelscope | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Subpackages | |||||
| ----------- | |||||
| .. toctree:: | |||||
| :maxdepth: 4 | |||||
| modelscope.fileio | |||||
| modelscope.models | |||||
| modelscope.pipelines | |||||
| modelscope.preprocessors | |||||
| modelscope.pydatasets | |||||
| modelscope.trainers | |||||
| modelscope.utils | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.version module | |||||
| ------------------------- | |||||
| .. automodule:: modelscope.version | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,7 +1,7 @@ | |||||
| maas\_lib.pipelines.nlp package | |||||
| modelscope.trainers.nlp package | |||||
| =============================== | =============================== | ||||
| .. automodule:: maas_lib.pipelines.nlp | |||||
| .. automodule:: modelscope.trainers.nlp | |||||
| :members: | :members: | ||||
| :undoc-members: | :undoc-members: | ||||
| :show-inheritance: | :show-inheritance: | ||||
| @@ -9,10 +9,10 @@ maas\_lib.pipelines.nlp package | |||||
| Submodules | Submodules | ||||
| ---------- | ---------- | ||||
| maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module | |||||
| ----------------------------------------------------------------- | |||||
| modelscope.trainers.nlp.sequence\_classification\_trainer module | |||||
| ---------------------------------------------------------------- | |||||
| .. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline | |||||
| .. automodule:: modelscope.trainers.nlp.sequence_classification_trainer | |||||
| :members: | :members: | ||||
| :undoc-members: | :undoc-members: | ||||
| :show-inheritance: | :show-inheritance: | ||||
| @@ -1,7 +1,7 @@ | |||||
| maas\_lib.pipelines package | |||||
| modelscope.trainers package | |||||
| =========================== | =========================== | ||||
| .. automodule:: maas_lib.pipelines | |||||
| .. automodule:: modelscope.trainers | |||||
| :members: | :members: | ||||
| :undoc-members: | :undoc-members: | ||||
| :show-inheritance: | :show-inheritance: | ||||
| @@ -12,25 +12,23 @@ Subpackages | |||||
| .. toctree:: | .. toctree:: | ||||
| :maxdepth: 4 | :maxdepth: 4 | ||||
| maas_lib.pipelines.cv | |||||
| maas_lib.pipelines.multi_modal | |||||
| maas_lib.pipelines.nlp | |||||
| modelscope.trainers.nlp | |||||
| Submodules | Submodules | ||||
| ---------- | ---------- | ||||
| maas\_lib.pipelines.base module | |||||
| modelscope.trainers.base module | |||||
| ------------------------------- | ------------------------------- | ||||
| .. automodule:: maas_lib.pipelines.base | |||||
| .. automodule:: modelscope.trainers.base | |||||
| :members: | :members: | ||||
| :undoc-members: | :undoc-members: | ||||
| :show-inheritance: | :show-inheritance: | ||||
| maas\_lib.pipelines.builder module | |||||
| modelscope.trainers.builder module | |||||
| ---------------------------------- | ---------------------------------- | ||||
| .. automodule:: maas_lib.pipelines.builder | |||||
| .. automodule:: modelscope.trainers.builder | |||||
| :members: | :members: | ||||
| :undoc-members: | :undoc-members: | ||||
| :show-inheritance: | :show-inheritance: | ||||
| @@ -0,0 +1,66 @@ | |||||
| modelscope.utils package | |||||
| ======================== | |||||
| .. automodule:: modelscope.utils | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| Submodules | |||||
| ---------- | |||||
| modelscope.utils.config module | |||||
| ------------------------------ | |||||
| .. automodule:: modelscope.utils.config | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.constant module | |||||
| -------------------------------- | |||||
| .. automodule:: modelscope.utils.constant | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.hub module | |||||
| --------------------------- | |||||
| .. automodule:: modelscope.utils.hub | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.logger module | |||||
| ------------------------------ | |||||
| .. automodule:: modelscope.utils.logger | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.pymod module | |||||
| ----------------------------- | |||||
| .. automodule:: modelscope.utils.pymod | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.registry module | |||||
| -------------------------------- | |||||
| .. automodule:: modelscope.utils.registry | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| modelscope.utils.type\_assert module | |||||
| ------------------------------------ | |||||
| .. automodule:: modelscope.utils.type_assert | |||||
| :members: | |||||
| :undoc-members: | |||||
| :show-inheritance: | |||||
| @@ -1,7 +1,7 @@ | |||||
| maas_lib | |||||
| ======== | |||||
| modelscope | |||||
| ========== | |||||
| .. toctree:: | .. toctree:: | ||||
| :maxdepth: 4 | :maxdepth: 4 | ||||
| maas_lib | |||||
| modelscope | |||||
| @@ -18,10 +18,10 @@ import sphinx_rtd_theme | |||||
| sys.path.insert(0, os.path.abspath('../../')) | sys.path.insert(0, os.path.abspath('../../')) | ||||
| # -- Project information ----------------------------------------------------- | # -- Project information ----------------------------------------------------- | ||||
| project = 'maas_lib' | |||||
| copyright = '2022-2023, Alibaba MaaS' | |||||
| author = 'maas_lib Authors' | |||||
| version_file = '../../maas_lib/version.py' | |||||
| project = 'modelscope' | |||||
| copyright = '2022-2023, Alibaba ModelScope' | |||||
| author = 'modelscope Authors' | |||||
| version_file = '../../modelscope/version.py' | |||||
| def get_version(): | def get_version(): | ||||
| @@ -88,7 +88,7 @@ html_static_path = ['_static'] | |||||
| # -- Options for HTMLHelp output --------------------------------------------- | # -- Options for HTMLHelp output --------------------------------------------- | ||||
| # Output file base name for HTML help builder. | # Output file base name for HTML help builder. | ||||
| htmlhelp_basename = 'maas_lib_doc' | |||||
| htmlhelp_basename = 'modelscope_doc' | |||||
| # -- Extension configuration ------------------------------------------------- | # -- Extension configuration ------------------------------------------------- | ||||
| # Ignore >>> when copying code | # Ignore >>> when copying code | ||||
| @@ -1,11 +1,11 @@ | |||||
| .. maas_lib documentation file, | |||||
| .. modelscope documentation file, | |||||
| You can adapt this file completely to your liking, but it should at least | You can adapt this file completely to your liking, but it should at least | ||||
| contain the root `toctree` directive. | contain the root `toctree` directive. | ||||
| MaasLib DOCUMENTATION | |||||
| ModelScope DOCUMENTATION | |||||
| ======================================= | ======================================= | ||||
| MaasLib doc | |||||
| ModelScope doc | |||||
| .. toctree:: | .. toctree:: | ||||
| :maxdepth: 2 | :maxdepth: 2 | ||||
| @@ -30,11 +30,11 @@ MaasLib doc | |||||
| :maxdepth: 10 | :maxdepth: 10 | ||||
| :caption: API Doc | :caption: API Doc | ||||
| api/maas_lib.preprocessors | |||||
| api/maas_lib.models | |||||
| api/maas_lib.pipelines | |||||
| api/maas_lib.fileio | |||||
| api/maas_lib.utils | |||||
| api/modelscope.preprocessors | |||||
| api/modelscope.models | |||||
| api/modelscope.pipelines | |||||
| api/modelscope.fileio | |||||
| api/modelscope.utils | |||||
| Indices and tables | Indices and tables | ||||
| @@ -5,39 +5,39 @@ | |||||
| 安装完成后,执行如下命令为maas library创建对应的python环境。 | 安装完成后,执行如下命令为maas library创建对应的python环境。 | ||||
| ```shell | ```shell | ||||
| conda create -n maas python=3.6 | |||||
| conda activate maas | |||||
| conda create -n modelscope python=3.6 | |||||
| conda activate modelscope | |||||
| ``` | ``` | ||||
| 检查python和pip命令是否切换到conda环境下。 | 检查python和pip命令是否切换到conda环境下。 | ||||
| ```shell | ```shell | ||||
| which python | which python | ||||
| # ~/workspace/anaconda3/envs/maas/bin/python | |||||
| # ~/workspace/anaconda3/envs/modelscope/bin/python | |||||
| which pip | which pip | ||||
| # ~/workspace/anaconda3/envs/maas/bin/pip | |||||
| # ~/workspace/anaconda3/envs/modelscope/bin/pip | |||||
| ``` | ``` | ||||
| 注: 本项目只支持`python3`环境,请勿使用python2环境。 | 注: 本项目只支持`python3`环境,请勿使用python2环境。 | ||||
| ## 第三方依赖安装 | ## 第三方依赖安装 | ||||
| MaaS Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架: | |||||
| ModelScope Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架: | |||||
| * [Pytorch安装指导](https://pytorch.org/get-started/locally/) | * [Pytorch安装指导](https://pytorch.org/get-started/locally/) | ||||
| * [Tensorflow安装指导](https://www.tensorflow.org/install/pip) | * [Tensorflow安装指导](https://www.tensorflow.org/install/pip) | ||||
| ## MaaS library 安装 | |||||
| ## ModelScope library 安装 | |||||
| 注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 | 注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 | ||||
| ### pip安装 | ### pip安装 | ||||
| ```shell | ```shell | ||||
| pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/maas.txt | |||||
| pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/modelscope.txt | |||||
| ``` | ``` | ||||
| 安装成功后,可以执行如下命令进行验证安装是否正确 | 安装成功后,可以执行如下命令进行验证安装是否正确 | ||||
| ```shell | ```shell | ||||
| python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
| python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
| ``` | ``` | ||||
| @@ -45,11 +45,11 @@ python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting | |||||
| 适合本地开发调试使用,修改源码后可以直接执行 | 适合本地开发调试使用,修改源码后可以直接执行 | ||||
| ```shell | ```shell | ||||
| git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git maaslib | |||||
| git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git modelscope | |||||
| git fetch origin master | git fetch origin master | ||||
| git checkout master | git checkout master | ||||
| cd maaslib | |||||
| cd modelscope | |||||
| #安装依赖 | #安装依赖 | ||||
| pip install -r requirements.txt | pip install -r requirements.txt | ||||
| @@ -60,7 +60,7 @@ export PYTHONPATH=`pwd` | |||||
| 安装成功后,可以执行如下命令进行验证安装是否正确 | 安装成功后,可以执行如下命令进行验证安装是否正确 | ||||
| ```shell | ```shell | ||||
| python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
| python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
| ``` | ``` | ||||
| @@ -79,8 +79,8 @@ pipeline函数提供了简洁的推理接口,示例如下, 更多pipeline介 | |||||
| ```python | ```python | ||||
| import cv2 | import cv2 | ||||
| import os.path as osp | import os.path as osp | ||||
| from maas_lib.pipelines import pipeline | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from modelscope.pipelines import pipeline | |||||
| from modelscope.utils.constant import Tasks | |||||
| # 根据任务名创建pipeline | # 根据任务名创建pipeline | ||||
| img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') | img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') | ||||
| @@ -99,9 +99,9 @@ print(f'Output written to {osp.abspath("result.png")}') | |||||
| ```python | ```python | ||||
| import cv2 | import cv2 | ||||
| import os.path as osp | import os.path as osp | ||||
| from maas_lib.pipelines import pipeline | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from maas_lib.pydatasets import PyDataset | |||||
| from modelscope.pipelines import pipeline | |||||
| from modelscope.utils.constant import Tasks | |||||
| from modelscope.pydatasets import PyDataset | |||||
| # 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 | # 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 | ||||
| input_location = [ | input_location = [ | ||||
| @@ -19,7 +19,7 @@ | |||||
| 1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象 | 1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象 | ||||
| 执行如下python代码 | 执行如下python代码 | ||||
| ```python | ```python | ||||
| >>> from maas_lib.pipelines import pipeline | |||||
| >>> from modelscope.pipelines import pipeline | |||||
| >>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') | >>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') | ||||
| ``` | ``` | ||||
| @@ -65,8 +65,8 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz | |||||
| 创建tokenizer和模型 | 创建tokenizer和模型 | ||||
| ```python | ```python | ||||
| >>> from maas_lib.models import Model | |||||
| >>> from maas_lib.preprocessors import SequenceClassificationPreprocessor | |||||
| >>> from modelscope.models import Model | |||||
| >>> from modelscope.preprocessors import SequenceClassificationPreprocessor | |||||
| >>> model = Model.from_pretrained('damo/bert-base-sst2') | >>> model = Model.from_pretrained('damo/bert-base-sst2') | ||||
| >>> tokenizer = SequenceClassificationPreprocessor( | >>> tokenizer = SequenceClassificationPreprocessor( | ||||
| model.model_dir, first_sequence='sentence', second_sequence=None) | model.model_dir, first_sequence='sentence', second_sequence=None) | ||||
| @@ -74,7 +74,7 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz | |||||
| 使用tokenizer和模型对象创建pipeline | 使用tokenizer和模型对象创建pipeline | ||||
| ```python | ```python | ||||
| >>> from maas_lib.pipelines import pipeline | |||||
| >>> from modelscope.pipelines import pipeline | |||||
| >>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) | >>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) | ||||
| >>> semantic_cls("Hello world!") | >>> semantic_cls("Hello world!") | ||||
| ``` | ``` | ||||
| @@ -1,48 +0,0 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | |||||
| from maas_lib.utils.constant import Tasks | |||||
| DEFAULT_MODEL_FOR_PIPELINE = { | |||||
| # TaskName: (pipeline_module_name, model_repo) | |||||
| Tasks.image_matting: ('image-matting', 'damo/image-matting-person'), | |||||
| Tasks.text_classification: | |||||
| ('bert-sentiment-analysis', 'damo/bert-base-sst2'), | |||||
| Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'), | |||||
| Tasks.image_captioning: ('ofa', None), | |||||
| } | |||||
| def add_default_pipeline_info(task: str, | |||||
| model_name: str, | |||||
| modelhub_name: str = None, | |||||
| overwrite: bool = False): | |||||
| """ Add default model for a task. | |||||
| Args: | |||||
| task (str): task name. | |||||
| model_name (str): model_name. | |||||
| modelhub_name (str): name for default modelhub. | |||||
| overwrite (bool): overwrite default info. | |||||
| """ | |||||
| if not overwrite: | |||||
| assert task not in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
| f'task {task} already has default model.' | |||||
| DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name) | |||||
| def get_default_pipeline_info(task): | |||||
| """ Get default info for certain task. | |||||
| Args: | |||||
| task (str): task name. | |||||
| Return: | |||||
| A tuple: first element is pipeline name(model_name), second element | |||||
| is modelhub name. | |||||
| """ | |||||
| assert task in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
| f'No default pipeline is registered for Task {task}' | |||||
| pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task] | |||||
| return pipeline_name, default_model | |||||
| @@ -7,10 +7,10 @@ from typing import Dict, List, Tuple, Union | |||||
| from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
| from maas_hub.snapshot_download import snapshot_download | from maas_hub.snapshot_download import snapshot_download | ||||
| from maas_lib.models.builder import build_model | |||||
| from maas_lib.utils.config import Config | |||||
| from maas_lib.utils.constant import CONFIGFILE | |||||
| from maas_lib.utils.hub import get_model_cache_dir | |||||
| from modelscope.models.builder import build_model | |||||
| from modelscope.utils.config import Config | |||||
| from modelscope.utils.constant import CONFIGFILE | |||||
| from modelscope.utils.hub import get_model_cache_dir | |||||
| Tensor = Union['torch.Tensor', 'tf.Tensor'] | Tensor = Union['torch.Tensor', 'tf.Tensor'] | ||||
| @@ -1,7 +1,7 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| from maas_lib.utils.config import ConfigDict | |||||
| from maas_lib.utils.registry import Registry, build_from_cfg | |||||
| from modelscope.utils.config import ConfigDict | |||||
| from modelscope.utils.registry import Registry, build_from_cfg | |||||
| MODELS = Registry('models') | MODELS = Registry('models') | ||||
| @@ -2,7 +2,7 @@ from typing import Any, Dict | |||||
| import numpy as np | import numpy as np | ||||
| from maas_lib.utils.constant import Tasks | |||||
| from modelscope.utils.constant import Tasks | |||||
| from ..base import Model | from ..base import Model | ||||
| from ..builder import MODELS | from ..builder import MODELS | ||||
| @@ -1,6 +1,6 @@ | |||||
| from typing import Any, Dict | from typing import Any, Dict | ||||
| from maas_lib.utils.constant import Tasks | |||||
| from modelscope.utils.constant import Tasks | |||||
| from ..base import Model, Tensor | from ..base import Model, Tensor | ||||
| from ..builder import MODELS | from ..builder import MODELS | ||||
| @@ -6,11 +6,11 @@ from typing import Any, Dict, Generator, List, Union | |||||
| from maas_hub.snapshot_download import snapshot_download | from maas_hub.snapshot_download import snapshot_download | ||||
| from maas_lib.models.base import Model | |||||
| from maas_lib.preprocessors import Preprocessor | |||||
| from maas_lib.pydatasets import PyDataset | |||||
| from maas_lib.utils.config import Config | |||||
| from maas_lib.utils.hub import get_model_cache_dir | |||||
| from modelscope.models.base import Model | |||||
| from modelscope.preprocessors import Preprocessor | |||||
| from modelscope.pydatasets import PyDataset | |||||
| from modelscope.utils.config import Config | |||||
| from modelscope.utils.hub import get_model_cache_dir | |||||
| from .util import is_model_name | from .util import is_model_name | ||||
| Tensor = Union['torch.Tensor', 'tf.Tensor'] | Tensor = Union['torch.Tensor', 'tf.Tensor'] | ||||
| @@ -75,7 +75,7 @@ class Pipeline(ABC): | |||||
| def __call__(self, input: Union[Input, List[Input]], *args, | def __call__(self, input: Union[Input, List[Input]], *args, | ||||
| **post_kwargs) -> Union[Dict[str, Any], Generator]: | **post_kwargs) -> Union[Dict[str, Any], Generator]: | ||||
| # model provider should leave it as it is | # model provider should leave it as it is | ||||
| # maas library developer will handle this function | |||||
| # modelscope library developer will handle this function | |||||
| # simple showcase, need to support iterator type for both tensorflow and pytorch | # simple showcase, need to support iterator type for both tensorflow and pytorch | ||||
| # input_dict = self._handle_input(input) | # input_dict = self._handle_input(input) | ||||
| @@ -6,16 +6,26 @@ from typing import Union | |||||
| import json | import json | ||||
| from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
| from maas_lib.models.base import Model | |||||
| from maas_lib.utils.config import Config, ConfigDict | |||||
| from maas_lib.utils.constant import CONFIGFILE, Tasks | |||||
| from maas_lib.utils.registry import Registry, build_from_cfg | |||||
| from modelscope.models.base import Model | |||||
| from modelscope.utils.config import Config, ConfigDict | |||||
| from modelscope.utils.constant import CONFIGFILE, Tasks | |||||
| from modelscope.utils.registry import Registry, build_from_cfg | |||||
| from .base import Pipeline | from .base import Pipeline | ||||
| from .default import DEFAULT_MODEL_FOR_PIPELINE, get_default_pipeline_info | |||||
| from .util import is_model_name | from .util import is_model_name | ||||
| PIPELINES = Registry('pipelines') | PIPELINES = Registry('pipelines') | ||||
| DEFAULT_MODEL_FOR_PIPELINE = { | |||||
| # TaskName: (pipeline_module_name, model_repo) | |||||
| Tasks.image_matting: ('image-matting', 'damo/image-matting-person'), | |||||
| Tasks.text_classification: | |||||
| ('bert-sentiment-analysis', 'damo/bert-base-sst2'), | |||||
| Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'), | |||||
| Tasks.image_captioning: ('ofa', None), | |||||
| Tasks.image_generation: | |||||
| ('cv_unet_person-image-cartoon', 'damo/cv_unet_image-matting_damo'), | |||||
| } | |||||
| def build_pipeline(cfg: ConfigDict, | def build_pipeline(cfg: ConfigDict, | ||||
| task_name: str = None, | task_name: str = None, | ||||
| @@ -84,3 +94,42 @@ def pipeline(task: str = None, | |||||
| cfg.preprocessor = preprocessor | cfg.preprocessor = preprocessor | ||||
| return build_pipeline(cfg, task_name=task) | return build_pipeline(cfg, task_name=task) | ||||
| def add_default_pipeline_info(task: str, | |||||
| model_name: str, | |||||
| modelhub_name: str = None, | |||||
| overwrite: bool = False): | |||||
| """ Add default model for a task. | |||||
| Args: | |||||
| task (str): task name. | |||||
| model_name (str): model_name. | |||||
| modelhub_name (str): name for default modelhub. | |||||
| overwrite (bool): overwrite default info. | |||||
| """ | |||||
| if not overwrite: | |||||
| assert task not in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
| f'task {task} already has default model.' | |||||
| DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name) | |||||
| def get_default_pipeline_info(task): | |||||
| """ Get default info for certain task. | |||||
| Args: | |||||
| task (str): task name. | |||||
| Return: | |||||
| A tuple: first element is pipeline name(model_name), second element | |||||
| is modelhub name. | |||||
| """ | |||||
| if task not in DEFAULT_MODEL_FOR_PIPELINE: | |||||
| # support pipeline which does not register default model | |||||
| pipeline_name = list(PIPELINES.modules[task].keys())[0] | |||||
| default_model = None | |||||
| else: | |||||
| pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task] | |||||
| return pipeline_name, default_model | |||||
| @@ -6,14 +6,14 @@ import numpy as np | |||||
| import PIL | import PIL | ||||
| import tensorflow as tf | import tensorflow as tf | ||||
| from maas_lib.models.cv.cartoon.facelib.facer import FaceAna | |||||
| from maas_lib.models.cv.cartoon.mtcnn_pytorch.src.align_trans import ( | |||||
| from modelscope.models.cv.cartoon.facelib.facer import FaceAna | |||||
| from modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans import ( | |||||
| get_reference_facial_points, warp_and_crop_face) | get_reference_facial_points, warp_and_crop_face) | ||||
| from maas_lib.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size | |||||
| from maas_lib.pipelines.base import Input | |||||
| from maas_lib.preprocessors import load_image | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from maas_lib.utils.logger import get_logger | |||||
| from modelscope.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size | |||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.preprocessors import load_image | |||||
| from modelscope.utils.constant import Tasks | |||||
| from modelscope.utils.logger import get_logger | |||||
| from ..base import Pipeline | from ..base import Pipeline | ||||
| from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
| @@ -5,10 +5,10 @@ import cv2 | |||||
| import numpy as np | import numpy as np | ||||
| import PIL | import PIL | ||||
| from maas_lib.pipelines.base import Input | |||||
| from maas_lib.preprocessors import load_image | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from maas_lib.utils.logger import get_logger | |||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.preprocessors import load_image | |||||
| from modelscope.utils.constant import Tasks | |||||
| from modelscope.utils.logger import get_logger | |||||
| from ..base import Pipeline | from ..base import Pipeline | ||||
| from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
| @@ -4,10 +4,10 @@ import numpy as np | |||||
| import torch | import torch | ||||
| from PIL import Image | from PIL import Image | ||||
| from maas_lib.pipelines.base import Input | |||||
| from maas_lib.preprocessors import load_image | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from maas_lib.utils.logger import get_logger | |||||
| from modelscope.pipelines.base import Input | |||||
| from modelscope.preprocessors import load_image | |||||
| from modelscope.utils.constant import Tasks | |||||
| from modelscope.utils.logger import get_logger | |||||
| from ..base import Pipeline | from ..base import Pipeline | ||||
| from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
| @@ -5,9 +5,9 @@ from typing import Any, Dict, Union | |||||
| import json | import json | ||||
| import numpy as np | import numpy as np | ||||
| from maas_lib.models.nlp import BertForSequenceClassification | |||||
| from maas_lib.preprocessors import SequenceClassificationPreprocessor | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from modelscope.models.nlp import BertForSequenceClassification | |||||
| from modelscope.preprocessors import SequenceClassificationPreprocessor | |||||
| from modelscope.utils.constant import Tasks | |||||
| from ...models import Model | from ...models import Model | ||||
| from ..base import Input, Pipeline | from ..base import Input, Pipeline | ||||
| from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
| @@ -1,9 +1,9 @@ | |||||
| from typing import Dict, Optional, Union | from typing import Dict, Optional, Union | ||||
| from maas_lib.models import Model | |||||
| from maas_lib.models.nlp import PalmForTextGenerationModel | |||||
| from maas_lib.preprocessors import TextGenerationPreprocessor | |||||
| from maas_lib.utils.constant import Tasks | |||||
| from modelscope.models import Model | |||||
| from modelscope.models.nlp import PalmForTextGenerationModel | |||||
| from modelscope.preprocessors import TextGenerationPreprocessor | |||||
| from modelscope.utils.constant import Tasks | |||||
| from ..base import Pipeline, Tensor | from ..base import Pipeline, Tensor | ||||
| from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
| @@ -5,7 +5,7 @@ import os.path as osp | |||||
| import json | import json | ||||
| from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
| from maas_lib.utils.constant import CONFIGFILE | |||||
| from modelscope.utils.constant import CONFIGFILE | |||||
| def is_model_name(model): | def is_model_name(model): | ||||
| @@ -1,8 +1,8 @@ | |||||
| # Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
| from maas_lib.utils.config import ConfigDict | |||||
| from maas_lib.utils.constant import Fields | |||||
| from maas_lib.utils.registry import Registry, build_from_cfg | |||||
| from modelscope.utils.config import ConfigDict | |||||
| from modelscope.utils.constant import Fields | |||||
| from modelscope.utils.registry import Registry, build_from_cfg | |||||
| PREPROCESSORS = Registry('preprocessors') | PREPROCESSORS = Registry('preprocessors') | ||||
| @@ -4,8 +4,8 @@ from typing import Dict, Union | |||||
| from PIL import Image, ImageOps | from PIL import Image, ImageOps | ||||
| from maas_lib.fileio import File | |||||
| from maas_lib.utils.constant import Fields | |||||
| from modelscope.fileio import File | |||||
| from modelscope.utils.constant import Fields | |||||
| from .builder import PREPROCESSORS | from .builder import PREPROCESSORS | ||||
| @@ -5,8 +5,8 @@ from typing import Any, Dict, Union | |||||
| from transformers import AutoTokenizer | from transformers import AutoTokenizer | ||||
| from maas_lib.utils.constant import Fields, InputFields | |||||
| from maas_lib.utils.type_assert import type_assert | |||||
| from modelscope.utils.constant import Fields, InputFields | |||||
| from modelscope.utils.type_assert import type_assert | |||||
| from .base import Preprocessor | from .base import Preprocessor | ||||
| from .builder import PREPROCESSORS | from .builder import PREPROCESSORS | ||||
| @@ -4,7 +4,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence, | |||||
| from datasets import Dataset, load_dataset | from datasets import Dataset, load_dataset | ||||
| from maas_lib.utils.logger import get_logger | |||||
| from modelscope.utils.logger import get_logger | |||||
| logger = get_logger() | logger = get_logger() | ||||
| @@ -52,7 +52,7 @@ class PyDataset: | |||||
| Mapping[str, Union[str, | Mapping[str, Union[str, | ||||
| Sequence[str]]]]] = None | Sequence[str]]]]] = None | ||||
| ) -> 'PyDataset': | ) -> 'PyDataset': | ||||
| """Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset. | |||||
| """Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. | |||||
| Args: | Args: | ||||
| path (str): Path or name of the dataset. | path (str): Path or name of the dataset. | ||||
| @@ -2,7 +2,7 @@ | |||||
| import argparse | import argparse | ||||
| from maas_lib.trainers import build_trainer | |||||
| from modelscope.trainers import build_trainer | |||||
| def parse_args(): | def parse_args(): | ||||
| @@ -2,7 +2,7 @@ | |||||
| import argparse | import argparse | ||||
| from maas_lib.trainers import build_trainer | |||||
| from modelscope.trainers import build_trainer | |||||
| def parse_args(): | def parse_args(): | ||||