1. refactor maaslib to modelscope 2. fix UT error 3. support pipeline which does not register default model Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8988388master
@@ -4,5 +4,5 @@ rm -rf build | |||||
# update api rst | # update api rst | ||||
#rm -rf source/api/ | #rm -rf source/api/ | ||||
#sphinx-apidoc --module-first -o source/api/ ../maas_lib/ | |||||
#sphinx-apidoc --module-first -o source/api/ ../modelscope/ | |||||
make html | make html |
@@ -1,3 +1,3 @@ | |||||
yapf -r -i maas_lib/ configs/ tests/ setup.py | |||||
isort -rc maas_lib/ configs/ tests/ setup.py | |||||
flake8 maas_lib/ configs/ tests/ setup.py | |||||
yapf -r -i modelscope/ configs/ tests/ setup.py | |||||
isort -rc modelscope/ configs/ tests/ setup.py | |||||
flake8 modelscope/ configs/ tests/ setup.py |
@@ -1,4 +1,4 @@ | |||||
Copyright 2022-2023 Alibaba MaaS. All rights reserved. | |||||
Copyright 2022-2023 Alibaba ModelScope. All rights reserved. | |||||
Apache License | Apache License | ||||
Version 2.0, January 2004 | Version 2.0, January 2004 | ||||
@@ -188,7 +188,7 @@ Copyright 2022-2023 Alibaba MaaS. All rights reserved. | |||||
same "printed page" as the copyright notice for easier | same "printed page" as the copyright notice for easier | ||||
identification within third-party archives. | identification within third-party archives. | ||||
Copyright 2020-2022 Alibaba MaaS. | |||||
Copyright 2020-2022 Alibaba ModelScope. | |||||
Licensed under the Apache License, Version 2.0 (the "License"); | Licensed under the Apache License, Version 2.0 (the "License"); | ||||
you may not use this file except in compliance with the License. | you may not use this file except in compliance with the License. | ||||
@@ -1 +1 @@ | |||||
recursive-include maas_lib/configs *.py | |||||
recursive-include modelscope/configs *.py |
@@ -1,6 +1,6 @@ | |||||
# Introduction | # Introduction | ||||
MaaS library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba. | |||||
ModelScope library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba. | |||||
# Design doc | # Design doc | ||||
@@ -1 +1 @@ | |||||
This folder will host example configs for each model supported by maas_lib. | |||||
This folder will host example configs for each model supported by modelscope. |
@@ -1,34 +0,0 @@ | |||||
maas\_lib.fileio.format package | |||||
=============================== | |||||
.. automodule:: maas_lib.fileio.format | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.fileio.format.base module | |||||
----------------------------------- | |||||
.. automodule:: maas_lib.fileio.format.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.fileio.format.json module | |||||
----------------------------------- | |||||
.. automodule:: maas_lib.fileio.format.json | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.fileio.format.yaml module | |||||
----------------------------------- | |||||
.. automodule:: maas_lib.fileio.format.yaml | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,34 +0,0 @@ | |||||
maas\_lib.fileio package | |||||
======================== | |||||
.. automodule:: maas_lib.fileio | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
maas_lib.fileio.format | |||||
Submodules | |||||
---------- | |||||
maas\_lib.fileio.file module | |||||
---------------------------- | |||||
.. automodule:: maas_lib.fileio.file | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.fileio.io module | |||||
-------------------------- | |||||
.. automodule:: maas_lib.fileio.io | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,18 +0,0 @@ | |||||
maas\_lib.models.nlp package | |||||
============================ | |||||
.. automodule:: maas_lib.models.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.models.nlp.sequence\_classification\_model module | |||||
----------------------------------------------------------- | |||||
.. automodule:: maas_lib.models.nlp.sequence_classification_model | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,34 +0,0 @@ | |||||
maas\_lib.models package | |||||
======================== | |||||
.. automodule:: maas_lib.models | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
maas_lib.models.nlp | |||||
Submodules | |||||
---------- | |||||
maas\_lib.models.base module | |||||
---------------------------- | |||||
.. automodule:: maas_lib.models.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.models.builder module | |||||
------------------------------- | |||||
.. automodule:: maas_lib.models.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,7 +0,0 @@ | |||||
maas\_lib.pipelines.audio package | |||||
================================= | |||||
.. automodule:: maas_lib.pipelines.audio | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,18 +0,0 @@ | |||||
maas\_lib.pipelines.cv package | |||||
============================== | |||||
.. automodule:: maas_lib.pipelines.cv | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.pipelines.cv.image\_matting module | |||||
-------------------------------------------- | |||||
.. automodule:: maas_lib.pipelines.cv.image_matting | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,7 +0,0 @@ | |||||
maas\_lib.pipelines.multi\_modal package | |||||
======================================== | |||||
.. automodule:: maas_lib.pipelines.multi_modal | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,50 +0,0 @@ | |||||
maas\_lib.preprocessors package | |||||
=============================== | |||||
.. automodule:: maas_lib.preprocessors | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.preprocessors.base module | |||||
----------------------------------- | |||||
.. automodule:: maas_lib.preprocessors.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.preprocessors.builder module | |||||
-------------------------------------- | |||||
.. automodule:: maas_lib.preprocessors.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.preprocessors.common module | |||||
------------------------------------- | |||||
.. automodule:: maas_lib.preprocessors.common | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.preprocessors.image module | |||||
------------------------------------ | |||||
.. automodule:: maas_lib.preprocessors.image | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.preprocessors.nlp module | |||||
---------------------------------- | |||||
.. automodule:: maas_lib.preprocessors.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,30 +0,0 @@ | |||||
maas\_lib package | |||||
================= | |||||
.. automodule:: maas_lib | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
maas_lib.fileio | |||||
maas_lib.models | |||||
maas_lib.pipelines | |||||
maas_lib.preprocessors | |||||
maas_lib.utils | |||||
Submodules | |||||
---------- | |||||
maas\_lib.version module | |||||
------------------------ | |||||
.. automodule:: maas_lib.version | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,18 +0,0 @@ | |||||
maas\_lib.trainers.nlp package | |||||
============================== | |||||
.. automodule:: maas_lib.trainers.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.trainers.nlp.sequence\_classification\_trainer module | |||||
--------------------------------------------------------------- | |||||
.. automodule:: maas_lib.trainers.nlp.sequence_classification_trainer | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,34 +0,0 @@ | |||||
maas\_lib.trainers package | |||||
========================== | |||||
.. automodule:: maas_lib.trainers | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
maas_lib.trainers.nlp | |||||
Submodules | |||||
---------- | |||||
maas\_lib.trainers.base module | |||||
------------------------------ | |||||
.. automodule:: maas_lib.trainers.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.trainers.builder module | |||||
--------------------------------- | |||||
.. automodule:: maas_lib.trainers.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,58 +0,0 @@ | |||||
maas\_lib.utils package | |||||
======================= | |||||
.. automodule:: maas_lib.utils | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
maas\_lib.utils.config module | |||||
----------------------------- | |||||
.. automodule:: maas_lib.utils.config | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.utils.constant module | |||||
------------------------------- | |||||
.. automodule:: maas_lib.utils.constant | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.utils.logger module | |||||
----------------------------- | |||||
.. automodule:: maas_lib.utils.logger | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.utils.pymod module | |||||
---------------------------- | |||||
.. automodule:: maas_lib.utils.pymod | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.utils.registry module | |||||
------------------------------- | |||||
.. automodule:: maas_lib.utils.registry | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
maas\_lib.utils.type\_assert module | |||||
----------------------------------- | |||||
.. automodule:: maas_lib.utils.type_assert | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,34 @@ | |||||
modelscope.fileio.format package | |||||
================================ | |||||
.. automodule:: modelscope.fileio.format | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.fileio.format.base module | |||||
------------------------------------ | |||||
.. automodule:: modelscope.fileio.format.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.fileio.format.json module | |||||
------------------------------------ | |||||
.. automodule:: modelscope.fileio.format.json | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.fileio.format.yaml module | |||||
------------------------------------ | |||||
.. automodule:: modelscope.fileio.format.yaml | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,34 @@ | |||||
modelscope.fileio package | |||||
========================= | |||||
.. automodule:: modelscope.fileio | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.fileio.format | |||||
Submodules | |||||
---------- | |||||
modelscope.fileio.file module | |||||
----------------------------- | |||||
.. automodule:: modelscope.fileio.file | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.fileio.io module | |||||
--------------------------- | |||||
.. automodule:: modelscope.fileio.io | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,18 @@ | |||||
modelscope.models.cv.cartoon.facelib.LK package | |||||
=============================================== | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.LK | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.models.cv.cartoon.facelib.LK.lk module | |||||
------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.LK.lk | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,50 @@ | |||||
modelscope.models.cv.cartoon.facelib package | |||||
============================================ | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.models.cv.cartoon.facelib.LK | |||||
Submodules | |||||
---------- | |||||
modelscope.models.cv.cartoon.facelib.config module | |||||
-------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.config | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.cv.cartoon.facelib.face\_detector module | |||||
---------------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.face_detector | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.cv.cartoon.facelib.face\_landmark module | |||||
---------------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.face_landmark | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.cv.cartoon.facelib.facer module | |||||
------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.facelib.facer | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,15 @@ | |||||
modelscope.models.cv.cartoon.mtcnn\_pytorch package | |||||
=================================================== | |||||
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.models.cv.cartoon.mtcnn_pytorch.src |
@@ -0,0 +1,26 @@ | |||||
modelscope.models.cv.cartoon.mtcnn\_pytorch.src package | |||||
======================================================= | |||||
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.models.cv.cartoon.mtcnn\_pytorch.src.align\_trans module | |||||
------------------------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.cv.cartoon.mtcnn\_pytorch.src.matlab\_cp2tform module | |||||
----------------------------------------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.matlab_cp2tform | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,27 @@ | |||||
modelscope.models.cv.cartoon package | |||||
==================================== | |||||
.. automodule:: modelscope.models.cv.cartoon | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.models.cv.cartoon.facelib | |||||
modelscope.models.cv.cartoon.mtcnn_pytorch | |||||
Submodules | |||||
---------- | |||||
modelscope.models.cv.cartoon.utils module | |||||
----------------------------------------- | |||||
.. automodule:: modelscope.models.cv.cartoon.utils | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,15 @@ | |||||
modelscope.models.cv package | |||||
============================ | |||||
.. automodule:: modelscope.models.cv | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.models.cv.cartoon |
@@ -0,0 +1,26 @@ | |||||
modelscope.models.nlp package | |||||
============================= | |||||
.. automodule:: modelscope.models.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.models.nlp.sequence\_classification\_model module | |||||
------------------------------------------------------------ | |||||
.. automodule:: modelscope.models.nlp.sequence_classification_model | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.nlp.text\_generation\_model module | |||||
---------------------------------------------------- | |||||
.. automodule:: modelscope.models.nlp.text_generation_model | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,35 @@ | |||||
modelscope.models package | |||||
========================= | |||||
.. automodule:: modelscope.models | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.models.cv | |||||
modelscope.models.nlp | |||||
Submodules | |||||
---------- | |||||
modelscope.models.base module | |||||
----------------------------- | |||||
.. automodule:: modelscope.models.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.models.builder module | |||||
-------------------------------- | |||||
.. automodule:: modelscope.models.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,7 @@ | |||||
modelscope.pipelines.audio package | |||||
================================== | |||||
.. automodule:: modelscope.pipelines.audio | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,26 @@ | |||||
modelscope.pipelines.cv package | |||||
=============================== | |||||
.. automodule:: modelscope.pipelines.cv | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.pipelines.cv.image\_cartoon\_pipeline module | |||||
------------------------------------------------------- | |||||
.. automodule:: modelscope.pipelines.cv.image_cartoon_pipeline | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.pipelines.cv.image\_matting\_pipeline module | |||||
------------------------------------------------------- | |||||
.. automodule:: modelscope.pipelines.cv.image_matting_pipeline | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,18 @@ | |||||
modelscope.pipelines.multi\_modal package | |||||
========================================= | |||||
.. automodule:: modelscope.pipelines.multi_modal | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.pipelines.multi\_modal.image\_captioning module | |||||
---------------------------------------------------------- | |||||
.. automodule:: modelscope.pipelines.multi_modal.image_captioning | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,26 @@ | |||||
modelscope.pipelines.nlp package | |||||
================================ | |||||
.. automodule:: modelscope.pipelines.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.pipelines.nlp.sequence\_classification\_pipeline module | |||||
------------------------------------------------------------------ | |||||
.. automodule:: modelscope.pipelines.nlp.sequence_classification_pipeline | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.pipelines.nlp.text\_generation\_pipeline module | |||||
---------------------------------------------------------- | |||||
.. automodule:: modelscope.pipelines.nlp.text_generation_pipeline | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,53 @@ | |||||
modelscope.pipelines package | |||||
============================ | |||||
.. automodule:: modelscope.pipelines | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.pipelines.audio | |||||
modelscope.pipelines.cv | |||||
modelscope.pipelines.multi_modal | |||||
modelscope.pipelines.nlp | |||||
Submodules | |||||
---------- | |||||
modelscope.pipelines.base module | |||||
-------------------------------- | |||||
.. automodule:: modelscope.pipelines.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.pipelines.builder module | |||||
----------------------------------- | |||||
.. automodule:: modelscope.pipelines.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.pipelines.default module | |||||
----------------------------------- | |||||
.. automodule:: modelscope.pipelines.default | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.pipelines.util module | |||||
-------------------------------- | |||||
.. automodule:: modelscope.pipelines.util | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,50 @@ | |||||
modelscope.preprocessors package | |||||
================================ | |||||
.. automodule:: modelscope.preprocessors | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.preprocessors.base module | |||||
------------------------------------ | |||||
.. automodule:: modelscope.preprocessors.base | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.preprocessors.builder module | |||||
--------------------------------------- | |||||
.. automodule:: modelscope.preprocessors.builder | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.preprocessors.common module | |||||
-------------------------------------- | |||||
.. automodule:: modelscope.preprocessors.common | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.preprocessors.image module | |||||
------------------------------------- | |||||
.. automodule:: modelscope.preprocessors.image | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.preprocessors.nlp module | |||||
----------------------------------- | |||||
.. automodule:: modelscope.preprocessors.nlp | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,18 @@ | |||||
modelscope.pydatasets package | |||||
============================= | |||||
.. automodule:: modelscope.pydatasets | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.pydatasets.py\_dataset module | |||||
---------------------------------------- | |||||
.. automodule:: modelscope.pydatasets.py_dataset | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -0,0 +1,32 @@ | |||||
modelscope package | |||||
================== | |||||
.. automodule:: modelscope | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Subpackages | |||||
----------- | |||||
.. toctree:: | |||||
:maxdepth: 4 | |||||
modelscope.fileio | |||||
modelscope.models | |||||
modelscope.pipelines | |||||
modelscope.preprocessors | |||||
modelscope.pydatasets | |||||
modelscope.trainers | |||||
modelscope.utils | |||||
Submodules | |||||
---------- | |||||
modelscope.version module | |||||
------------------------- | |||||
.. automodule:: modelscope.version | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,7 +1,7 @@ | |||||
maas\_lib.pipelines.nlp package | |||||
modelscope.trainers.nlp package | |||||
=============================== | =============================== | ||||
.. automodule:: maas_lib.pipelines.nlp | |||||
.. automodule:: modelscope.trainers.nlp | |||||
:members: | :members: | ||||
:undoc-members: | :undoc-members: | ||||
:show-inheritance: | :show-inheritance: | ||||
@@ -9,10 +9,10 @@ maas\_lib.pipelines.nlp package | |||||
Submodules | Submodules | ||||
---------- | ---------- | ||||
maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module | |||||
----------------------------------------------------------------- | |||||
modelscope.trainers.nlp.sequence\_classification\_trainer module | |||||
---------------------------------------------------------------- | |||||
.. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline | |||||
.. automodule:: modelscope.trainers.nlp.sequence_classification_trainer | |||||
:members: | :members: | ||||
:undoc-members: | :undoc-members: | ||||
:show-inheritance: | :show-inheritance: |
@@ -1,7 +1,7 @@ | |||||
maas\_lib.pipelines package | |||||
modelscope.trainers package | |||||
=========================== | =========================== | ||||
.. automodule:: maas_lib.pipelines | |||||
.. automodule:: modelscope.trainers | |||||
:members: | :members: | ||||
:undoc-members: | :undoc-members: | ||||
:show-inheritance: | :show-inheritance: | ||||
@@ -12,25 +12,23 @@ Subpackages | |||||
.. toctree:: | .. toctree:: | ||||
:maxdepth: 4 | :maxdepth: 4 | ||||
maas_lib.pipelines.cv | |||||
maas_lib.pipelines.multi_modal | |||||
maas_lib.pipelines.nlp | |||||
modelscope.trainers.nlp | |||||
Submodules | Submodules | ||||
---------- | ---------- | ||||
maas\_lib.pipelines.base module | |||||
modelscope.trainers.base module | |||||
------------------------------- | ------------------------------- | ||||
.. automodule:: maas_lib.pipelines.base | |||||
.. automodule:: modelscope.trainers.base | |||||
:members: | :members: | ||||
:undoc-members: | :undoc-members: | ||||
:show-inheritance: | :show-inheritance: | ||||
maas\_lib.pipelines.builder module | |||||
modelscope.trainers.builder module | |||||
---------------------------------- | ---------------------------------- | ||||
.. automodule:: maas_lib.pipelines.builder | |||||
.. automodule:: modelscope.trainers.builder | |||||
:members: | :members: | ||||
:undoc-members: | :undoc-members: | ||||
:show-inheritance: | :show-inheritance: |
@@ -0,0 +1,66 @@ | |||||
modelscope.utils package | |||||
======================== | |||||
.. automodule:: modelscope.utils | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
Submodules | |||||
---------- | |||||
modelscope.utils.config module | |||||
------------------------------ | |||||
.. automodule:: modelscope.utils.config | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.constant module | |||||
-------------------------------- | |||||
.. automodule:: modelscope.utils.constant | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.hub module | |||||
--------------------------- | |||||
.. automodule:: modelscope.utils.hub | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.logger module | |||||
------------------------------ | |||||
.. automodule:: modelscope.utils.logger | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.pymod module | |||||
----------------------------- | |||||
.. automodule:: modelscope.utils.pymod | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.registry module | |||||
-------------------------------- | |||||
.. automodule:: modelscope.utils.registry | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: | |||||
modelscope.utils.type\_assert module | |||||
------------------------------------ | |||||
.. automodule:: modelscope.utils.type_assert | |||||
:members: | |||||
:undoc-members: | |||||
:show-inheritance: |
@@ -1,7 +1,7 @@ | |||||
maas_lib | |||||
======== | |||||
modelscope | |||||
========== | |||||
.. toctree:: | .. toctree:: | ||||
:maxdepth: 4 | :maxdepth: 4 | ||||
maas_lib | |||||
modelscope |
@@ -18,10 +18,10 @@ import sphinx_rtd_theme | |||||
sys.path.insert(0, os.path.abspath('../../')) | sys.path.insert(0, os.path.abspath('../../')) | ||||
# -- Project information ----------------------------------------------------- | # -- Project information ----------------------------------------------------- | ||||
project = 'maas_lib' | |||||
copyright = '2022-2023, Alibaba MaaS' | |||||
author = 'maas_lib Authors' | |||||
version_file = '../../maas_lib/version.py' | |||||
project = 'modelscope' | |||||
copyright = '2022-2023, Alibaba ModelScope' | |||||
author = 'modelscope Authors' | |||||
version_file = '../../modelscope/version.py' | |||||
def get_version(): | def get_version(): | ||||
@@ -88,7 +88,7 @@ html_static_path = ['_static'] | |||||
# -- Options for HTMLHelp output --------------------------------------------- | # -- Options for HTMLHelp output --------------------------------------------- | ||||
# Output file base name for HTML help builder. | # Output file base name for HTML help builder. | ||||
htmlhelp_basename = 'maas_lib_doc' | |||||
htmlhelp_basename = 'modelscope_doc' | |||||
# -- Extension configuration ------------------------------------------------- | # -- Extension configuration ------------------------------------------------- | ||||
# Ignore >>> when copying code | # Ignore >>> when copying code | ||||
@@ -1,11 +1,11 @@ | |||||
.. maas_lib documentation file, | |||||
.. modelscope documentation file, | |||||
You can adapt this file completely to your liking, but it should at least | You can adapt this file completely to your liking, but it should at least | ||||
contain the root `toctree` directive. | contain the root `toctree` directive. | ||||
MaasLib DOCUMENTATION | |||||
ModelScope DOCUMENTATION | |||||
======================================= | ======================================= | ||||
MaasLib doc | |||||
ModelScope doc | |||||
.. toctree:: | .. toctree:: | ||||
:maxdepth: 2 | :maxdepth: 2 | ||||
@@ -30,11 +30,11 @@ MaasLib doc | |||||
:maxdepth: 10 | :maxdepth: 10 | ||||
:caption: API Doc | :caption: API Doc | ||||
api/maas_lib.preprocessors | |||||
api/maas_lib.models | |||||
api/maas_lib.pipelines | |||||
api/maas_lib.fileio | |||||
api/maas_lib.utils | |||||
api/modelscope.preprocessors | |||||
api/modelscope.models | |||||
api/modelscope.pipelines | |||||
api/modelscope.fileio | |||||
api/modelscope.utils | |||||
Indices and tables | Indices and tables | ||||
@@ -5,39 +5,39 @@ | |||||
安装完成后,执行如下命令为maas library创建对应的python环境。 | 安装完成后,执行如下命令为maas library创建对应的python环境。 | ||||
```shell | ```shell | ||||
conda create -n maas python=3.6 | |||||
conda activate maas | |||||
conda create -n modelscope python=3.6 | |||||
conda activate modelscope | |||||
``` | ``` | ||||
检查python和pip命令是否切换到conda环境下。 | 检查python和pip命令是否切换到conda环境下。 | ||||
```shell | ```shell | ||||
which python | which python | ||||
# ~/workspace/anaconda3/envs/maas/bin/python | |||||
# ~/workspace/anaconda3/envs/modelscope/bin/python | |||||
which pip | which pip | ||||
# ~/workspace/anaconda3/envs/maas/bin/pip | |||||
# ~/workspace/anaconda3/envs/modelscope/bin/pip | |||||
``` | ``` | ||||
注: 本项目只支持`python3`环境,请勿使用python2环境。 | 注: 本项目只支持`python3`环境,请勿使用python2环境。 | ||||
## 第三方依赖安装 | ## 第三方依赖安装 | ||||
MaaS Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架: | |||||
ModelScope Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架: | |||||
* [Pytorch安装指导](https://pytorch.org/get-started/locally/) | * [Pytorch安装指导](https://pytorch.org/get-started/locally/) | ||||
* [Tensorflow安装指导](https://www.tensorflow.org/install/pip) | * [Tensorflow安装指导](https://www.tensorflow.org/install/pip) | ||||
## MaaS library 安装 | |||||
## ModelScope library 安装 | |||||
注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 | 注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 | ||||
### pip安装 | ### pip安装 | ||||
```shell | ```shell | ||||
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/maas.txt | |||||
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/modelscope.txt | |||||
``` | ``` | ||||
安装成功后,可以执行如下命令进行验证安装是否正确 | 安装成功后,可以执行如下命令进行验证安装是否正确 | ||||
```shell | ```shell | ||||
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
``` | ``` | ||||
@@ -45,11 +45,11 @@ python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting | |||||
适合本地开发调试使用,修改源码后可以直接执行 | 适合本地开发调试使用,修改源码后可以直接执行 | ||||
```shell | ```shell | ||||
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git maaslib | |||||
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git modelscope | |||||
git fetch origin master | git fetch origin master | ||||
git checkout master | git checkout master | ||||
cd maaslib | |||||
cd modelscope | |||||
#安装依赖 | #安装依赖 | ||||
pip install -r requirements.txt | pip install -r requirements.txt | ||||
@@ -60,7 +60,7 @@ export PYTHONPATH=`pwd` | |||||
安装成功后,可以执行如下命令进行验证安装是否正确 | 安装成功后,可以执行如下命令进行验证安装是否正确 | ||||
```shell | ```shell | ||||
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))" | |||||
``` | ``` | ||||
@@ -79,8 +79,8 @@ pipeline函数提供了简洁的推理接口,示例如下, 更多pipeline介 | |||||
```python | ```python | ||||
import cv2 | import cv2 | ||||
import os.path as osp | import os.path as osp | ||||
from maas_lib.pipelines import pipeline | |||||
from maas_lib.utils.constant import Tasks | |||||
from modelscope.pipelines import pipeline | |||||
from modelscope.utils.constant import Tasks | |||||
# 根据任务名创建pipeline | # 根据任务名创建pipeline | ||||
img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') | img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') | ||||
@@ -99,9 +99,9 @@ print(f'Output written to {osp.abspath("result.png")}') | |||||
```python | ```python | ||||
import cv2 | import cv2 | ||||
import os.path as osp | import os.path as osp | ||||
from maas_lib.pipelines import pipeline | |||||
from maas_lib.utils.constant import Tasks | |||||
from maas_lib.pydatasets import PyDataset | |||||
from modelscope.pipelines import pipeline | |||||
from modelscope.utils.constant import Tasks | |||||
from modelscope.pydatasets import PyDataset | |||||
# 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 | # 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 | ||||
input_location = [ | input_location = [ | ||||
@@ -19,7 +19,7 @@ | |||||
1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象 | 1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象 | ||||
执行如下python代码 | 执行如下python代码 | ||||
```python | ```python | ||||
>>> from maas_lib.pipelines import pipeline | |||||
>>> from modelscope.pipelines import pipeline | |||||
>>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') | >>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') | ||||
``` | ``` | ||||
@@ -65,8 +65,8 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz | |||||
创建tokenizer和模型 | 创建tokenizer和模型 | ||||
```python | ```python | ||||
>>> from maas_lib.models import Model | |||||
>>> from maas_lib.preprocessors import SequenceClassificationPreprocessor | |||||
>>> from modelscope.models import Model | |||||
>>> from modelscope.preprocessors import SequenceClassificationPreprocessor | |||||
>>> model = Model.from_pretrained('damo/bert-base-sst2') | >>> model = Model.from_pretrained('damo/bert-base-sst2') | ||||
>>> tokenizer = SequenceClassificationPreprocessor( | >>> tokenizer = SequenceClassificationPreprocessor( | ||||
model.model_dir, first_sequence='sentence', second_sequence=None) | model.model_dir, first_sequence='sentence', second_sequence=None) | ||||
@@ -74,7 +74,7 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz | |||||
使用tokenizer和模型对象创建pipeline | 使用tokenizer和模型对象创建pipeline | ||||
```python | ```python | ||||
>>> from maas_lib.pipelines import pipeline | |||||
>>> from modelscope.pipelines import pipeline | |||||
>>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) | >>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) | ||||
>>> semantic_cls("Hello world!") | >>> semantic_cls("Hello world!") | ||||
``` | ``` | ||||
@@ -1,48 +0,0 @@ | |||||
# Copyright (c) Alibaba, Inc. and its affiliates. | |||||
from maas_lib.utils.constant import Tasks | |||||
DEFAULT_MODEL_FOR_PIPELINE = { | |||||
# TaskName: (pipeline_module_name, model_repo) | |||||
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'), | |||||
Tasks.text_classification: | |||||
('bert-sentiment-analysis', 'damo/bert-base-sst2'), | |||||
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'), | |||||
Tasks.image_captioning: ('ofa', None), | |||||
} | |||||
def add_default_pipeline_info(task: str, | |||||
model_name: str, | |||||
modelhub_name: str = None, | |||||
overwrite: bool = False): | |||||
""" Add default model for a task. | |||||
Args: | |||||
task (str): task name. | |||||
model_name (str): model_name. | |||||
modelhub_name (str): name for default modelhub. | |||||
overwrite (bool): overwrite default info. | |||||
""" | |||||
if not overwrite: | |||||
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
f'task {task} already has default model.' | |||||
DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name) | |||||
def get_default_pipeline_info(task): | |||||
""" Get default info for certain task. | |||||
Args: | |||||
task (str): task name. | |||||
Return: | |||||
A tuple: first element is pipeline name(model_name), second element | |||||
is modelhub name. | |||||
""" | |||||
assert task in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
f'No default pipeline is registered for Task {task}' | |||||
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task] | |||||
return pipeline_name, default_model |
@@ -7,10 +7,10 @@ from typing import Dict, List, Tuple, Union | |||||
from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
from maas_hub.snapshot_download import snapshot_download | from maas_hub.snapshot_download import snapshot_download | ||||
from maas_lib.models.builder import build_model | |||||
from maas_lib.utils.config import Config | |||||
from maas_lib.utils.constant import CONFIGFILE | |||||
from maas_lib.utils.hub import get_model_cache_dir | |||||
from modelscope.models.builder import build_model | |||||
from modelscope.utils.config import Config | |||||
from modelscope.utils.constant import CONFIGFILE | |||||
from modelscope.utils.hub import get_model_cache_dir | |||||
Tensor = Union['torch.Tensor', 'tf.Tensor'] | Tensor = Union['torch.Tensor', 'tf.Tensor'] | ||||
@@ -1,7 +1,7 @@ | |||||
# Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
from maas_lib.utils.config import ConfigDict | |||||
from maas_lib.utils.registry import Registry, build_from_cfg | |||||
from modelscope.utils.config import ConfigDict | |||||
from modelscope.utils.registry import Registry, build_from_cfg | |||||
MODELS = Registry('models') | MODELS = Registry('models') | ||||
@@ -2,7 +2,7 @@ from typing import Any, Dict | |||||
import numpy as np | import numpy as np | ||||
from maas_lib.utils.constant import Tasks | |||||
from modelscope.utils.constant import Tasks | |||||
from ..base import Model | from ..base import Model | ||||
from ..builder import MODELS | from ..builder import MODELS | ||||
@@ -1,6 +1,6 @@ | |||||
from typing import Any, Dict | from typing import Any, Dict | ||||
from maas_lib.utils.constant import Tasks | |||||
from modelscope.utils.constant import Tasks | |||||
from ..base import Model, Tensor | from ..base import Model, Tensor | ||||
from ..builder import MODELS | from ..builder import MODELS | ||||
@@ -6,11 +6,11 @@ from typing import Any, Dict, Generator, List, Union | |||||
from maas_hub.snapshot_download import snapshot_download | from maas_hub.snapshot_download import snapshot_download | ||||
from maas_lib.models.base import Model | |||||
from maas_lib.preprocessors import Preprocessor | |||||
from maas_lib.pydatasets import PyDataset | |||||
from maas_lib.utils.config import Config | |||||
from maas_lib.utils.hub import get_model_cache_dir | |||||
from modelscope.models.base import Model | |||||
from modelscope.preprocessors import Preprocessor | |||||
from modelscope.pydatasets import PyDataset | |||||
from modelscope.utils.config import Config | |||||
from modelscope.utils.hub import get_model_cache_dir | |||||
from .util import is_model_name | from .util import is_model_name | ||||
Tensor = Union['torch.Tensor', 'tf.Tensor'] | Tensor = Union['torch.Tensor', 'tf.Tensor'] | ||||
@@ -75,7 +75,7 @@ class Pipeline(ABC): | |||||
def __call__(self, input: Union[Input, List[Input]], *args, | def __call__(self, input: Union[Input, List[Input]], *args, | ||||
**post_kwargs) -> Union[Dict[str, Any], Generator]: | **post_kwargs) -> Union[Dict[str, Any], Generator]: | ||||
# model provider should leave it as it is | # model provider should leave it as it is | ||||
# maas library developer will handle this function | |||||
# modelscope library developer will handle this function | |||||
# simple showcase, need to support iterator type for both tensorflow and pytorch | # simple showcase, need to support iterator type for both tensorflow and pytorch | ||||
# input_dict = self._handle_input(input) | # input_dict = self._handle_input(input) |
@@ -6,16 +6,26 @@ from typing import Union | |||||
import json | import json | ||||
from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
from maas_lib.models.base import Model | |||||
from maas_lib.utils.config import Config, ConfigDict | |||||
from maas_lib.utils.constant import CONFIGFILE, Tasks | |||||
from maas_lib.utils.registry import Registry, build_from_cfg | |||||
from modelscope.models.base import Model | |||||
from modelscope.utils.config import Config, ConfigDict | |||||
from modelscope.utils.constant import CONFIGFILE, Tasks | |||||
from modelscope.utils.registry import Registry, build_from_cfg | |||||
from .base import Pipeline | from .base import Pipeline | ||||
from .default import DEFAULT_MODEL_FOR_PIPELINE, get_default_pipeline_info | |||||
from .util import is_model_name | from .util import is_model_name | ||||
PIPELINES = Registry('pipelines') | PIPELINES = Registry('pipelines') | ||||
DEFAULT_MODEL_FOR_PIPELINE = { | |||||
# TaskName: (pipeline_module_name, model_repo) | |||||
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'), | |||||
Tasks.text_classification: | |||||
('bert-sentiment-analysis', 'damo/bert-base-sst2'), | |||||
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'), | |||||
Tasks.image_captioning: ('ofa', None), | |||||
Tasks.image_generation: | |||||
('cv_unet_person-image-cartoon', 'damo/cv_unet_image-matting_damo'), | |||||
} | |||||
def build_pipeline(cfg: ConfigDict, | def build_pipeline(cfg: ConfigDict, | ||||
task_name: str = None, | task_name: str = None, | ||||
@@ -84,3 +94,42 @@ def pipeline(task: str = None, | |||||
cfg.preprocessor = preprocessor | cfg.preprocessor = preprocessor | ||||
return build_pipeline(cfg, task_name=task) | return build_pipeline(cfg, task_name=task) | ||||
def add_default_pipeline_info(task: str, | |||||
model_name: str, | |||||
modelhub_name: str = None, | |||||
overwrite: bool = False): | |||||
""" Add default model for a task. | |||||
Args: | |||||
task (str): task name. | |||||
model_name (str): model_name. | |||||
modelhub_name (str): name for default modelhub. | |||||
overwrite (bool): overwrite default info. | |||||
""" | |||||
if not overwrite: | |||||
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \ | |||||
f'task {task} already has default model.' | |||||
DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name) | |||||
def get_default_pipeline_info(task): | |||||
""" Get default info for certain task. | |||||
Args: | |||||
task (str): task name. | |||||
Return: | |||||
A tuple: first element is pipeline name(model_name), second element | |||||
is modelhub name. | |||||
""" | |||||
if task not in DEFAULT_MODEL_FOR_PIPELINE: | |||||
# support pipeline which does not register default model | |||||
pipeline_name = list(PIPELINES.modules[task].keys())[0] | |||||
default_model = None | |||||
else: | |||||
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task] | |||||
return pipeline_name, default_model |
@@ -6,14 +6,14 @@ import numpy as np | |||||
import PIL | import PIL | ||||
import tensorflow as tf | import tensorflow as tf | ||||
from maas_lib.models.cv.cartoon.facelib.facer import FaceAna | |||||
from maas_lib.models.cv.cartoon.mtcnn_pytorch.src.align_trans import ( | |||||
from modelscope.models.cv.cartoon.facelib.facer import FaceAna | |||||
from modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans import ( | |||||
get_reference_facial_points, warp_and_crop_face) | get_reference_facial_points, warp_and_crop_face) | ||||
from maas_lib.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size | |||||
from maas_lib.pipelines.base import Input | |||||
from maas_lib.preprocessors import load_image | |||||
from maas_lib.utils.constant import Tasks | |||||
from maas_lib.utils.logger import get_logger | |||||
from modelscope.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size | |||||
from modelscope.pipelines.base import Input | |||||
from modelscope.preprocessors import load_image | |||||
from modelscope.utils.constant import Tasks | |||||
from modelscope.utils.logger import get_logger | |||||
from ..base import Pipeline | from ..base import Pipeline | ||||
from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
@@ -5,10 +5,10 @@ import cv2 | |||||
import numpy as np | import numpy as np | ||||
import PIL | import PIL | ||||
from maas_lib.pipelines.base import Input | |||||
from maas_lib.preprocessors import load_image | |||||
from maas_lib.utils.constant import Tasks | |||||
from maas_lib.utils.logger import get_logger | |||||
from modelscope.pipelines.base import Input | |||||
from modelscope.preprocessors import load_image | |||||
from modelscope.utils.constant import Tasks | |||||
from modelscope.utils.logger import get_logger | |||||
from ..base import Pipeline | from ..base import Pipeline | ||||
from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
@@ -4,10 +4,10 @@ import numpy as np | |||||
import torch | import torch | ||||
from PIL import Image | from PIL import Image | ||||
from maas_lib.pipelines.base import Input | |||||
from maas_lib.preprocessors import load_image | |||||
from maas_lib.utils.constant import Tasks | |||||
from maas_lib.utils.logger import get_logger | |||||
from modelscope.pipelines.base import Input | |||||
from modelscope.preprocessors import load_image | |||||
from modelscope.utils.constant import Tasks | |||||
from modelscope.utils.logger import get_logger | |||||
from ..base import Pipeline | from ..base import Pipeline | ||||
from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
@@ -5,9 +5,9 @@ from typing import Any, Dict, Union | |||||
import json | import json | ||||
import numpy as np | import numpy as np | ||||
from maas_lib.models.nlp import BertForSequenceClassification | |||||
from maas_lib.preprocessors import SequenceClassificationPreprocessor | |||||
from maas_lib.utils.constant import Tasks | |||||
from modelscope.models.nlp import BertForSequenceClassification | |||||
from modelscope.preprocessors import SequenceClassificationPreprocessor | |||||
from modelscope.utils.constant import Tasks | |||||
from ...models import Model | from ...models import Model | ||||
from ..base import Input, Pipeline | from ..base import Input, Pipeline | ||||
from ..builder import PIPELINES | from ..builder import PIPELINES |
@@ -1,9 +1,9 @@ | |||||
from typing import Dict, Optional, Union | from typing import Dict, Optional, Union | ||||
from maas_lib.models import Model | |||||
from maas_lib.models.nlp import PalmForTextGenerationModel | |||||
from maas_lib.preprocessors import TextGenerationPreprocessor | |||||
from maas_lib.utils.constant import Tasks | |||||
from modelscope.models import Model | |||||
from modelscope.models.nlp import PalmForTextGenerationModel | |||||
from modelscope.preprocessors import TextGenerationPreprocessor | |||||
from modelscope.utils.constant import Tasks | |||||
from ..base import Pipeline, Tensor | from ..base import Pipeline, Tensor | ||||
from ..builder import PIPELINES | from ..builder import PIPELINES | ||||
@@ -5,7 +5,7 @@ import os.path as osp | |||||
import json | import json | ||||
from maas_hub.file_download import model_file_download | from maas_hub.file_download import model_file_download | ||||
from maas_lib.utils.constant import CONFIGFILE | |||||
from modelscope.utils.constant import CONFIGFILE | |||||
def is_model_name(model): | def is_model_name(model): |
@@ -1,8 +1,8 @@ | |||||
# Copyright (c) Alibaba, Inc. and its affiliates. | # Copyright (c) Alibaba, Inc. and its affiliates. | ||||
from maas_lib.utils.config import ConfigDict | |||||
from maas_lib.utils.constant import Fields | |||||
from maas_lib.utils.registry import Registry, build_from_cfg | |||||
from modelscope.utils.config import ConfigDict | |||||
from modelscope.utils.constant import Fields | |||||
from modelscope.utils.registry import Registry, build_from_cfg | |||||
PREPROCESSORS = Registry('preprocessors') | PREPROCESSORS = Registry('preprocessors') | ||||
@@ -4,8 +4,8 @@ from typing import Dict, Union | |||||
from PIL import Image, ImageOps | from PIL import Image, ImageOps | ||||
from maas_lib.fileio import File | |||||
from maas_lib.utils.constant import Fields | |||||
from modelscope.fileio import File | |||||
from modelscope.utils.constant import Fields | |||||
from .builder import PREPROCESSORS | from .builder import PREPROCESSORS | ||||
@@ -5,8 +5,8 @@ from typing import Any, Dict, Union | |||||
from transformers import AutoTokenizer | from transformers import AutoTokenizer | ||||
from maas_lib.utils.constant import Fields, InputFields | |||||
from maas_lib.utils.type_assert import type_assert | |||||
from modelscope.utils.constant import Fields, InputFields | |||||
from modelscope.utils.type_assert import type_assert | |||||
from .base import Preprocessor | from .base import Preprocessor | ||||
from .builder import PREPROCESSORS | from .builder import PREPROCESSORS | ||||
@@ -4,7 +4,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence, | |||||
from datasets import Dataset, load_dataset | from datasets import Dataset, load_dataset | ||||
from maas_lib.utils.logger import get_logger | |||||
from modelscope.utils.logger import get_logger | |||||
logger = get_logger() | logger = get_logger() | ||||
@@ -52,7 +52,7 @@ class PyDataset: | |||||
Mapping[str, Union[str, | Mapping[str, Union[str, | ||||
Sequence[str]]]]] = None | Sequence[str]]]]] = None | ||||
) -> 'PyDataset': | ) -> 'PyDataset': | ||||
"""Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset. | |||||
"""Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset. | |||||
Args: | Args: | ||||
path (str): Path or name of the dataset. | path (str): Path or name of the dataset. |
@@ -2,7 +2,7 @@ | |||||
import argparse | import argparse | ||||
from maas_lib.trainers import build_trainer | |||||
from modelscope.trainers import build_trainer | |||||
def parse_args(): | def parse_args(): |
@@ -2,7 +2,7 @@ | |||||
import argparse | import argparse | ||||
from maas_lib.trainers import build_trainer | |||||
from modelscope.trainers import build_trainer | |||||
def parse_args(): | def parse_args(): |