Browse Source

[to #42373878] refactor maaslib to modelscope

1.  refactor maaslib to modelscope
2.  fix UT error
3.  support pipeline which does not register default model

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8988388
master
wenmeng.zwm 3 years ago
parent
commit
1f6b376599
100 changed files with 735 additions and 502 deletions
  1. +1
    -1
      .dev_scripts/build_docs.sh
  2. +3
    -3
      .dev_scripts/linter.sh
  3. +2
    -2
      LICENSE
  4. +1
    -1
      MANIFEST.in
  5. +1
    -1
      README.md
  6. +1
    -1
      configs/README.md
  7. +0
    -34
      docs/source/api/maas_lib.fileio.format.rst
  8. +0
    -34
      docs/source/api/maas_lib.fileio.rst
  9. +0
    -18
      docs/source/api/maas_lib.models.nlp.rst
  10. +0
    -34
      docs/source/api/maas_lib.models.rst
  11. +0
    -7
      docs/source/api/maas_lib.pipelines.audio.rst
  12. +0
    -18
      docs/source/api/maas_lib.pipelines.cv.rst
  13. +0
    -7
      docs/source/api/maas_lib.pipelines.multi_modal.rst
  14. +0
    -50
      docs/source/api/maas_lib.preprocessors.rst
  15. +0
    -30
      docs/source/api/maas_lib.rst
  16. +0
    -18
      docs/source/api/maas_lib.trainers.nlp.rst
  17. +0
    -34
      docs/source/api/maas_lib.trainers.rst
  18. +0
    -58
      docs/source/api/maas_lib.utils.rst
  19. +34
    -0
      docs/source/api/modelscope.fileio.format.rst
  20. +34
    -0
      docs/source/api/modelscope.fileio.rst
  21. +18
    -0
      docs/source/api/modelscope.models.cv.cartoon.facelib.LK.rst
  22. +50
    -0
      docs/source/api/modelscope.models.cv.cartoon.facelib.rst
  23. +15
    -0
      docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.rst
  24. +26
    -0
      docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.src.rst
  25. +27
    -0
      docs/source/api/modelscope.models.cv.cartoon.rst
  26. +15
    -0
      docs/source/api/modelscope.models.cv.rst
  27. +26
    -0
      docs/source/api/modelscope.models.nlp.rst
  28. +35
    -0
      docs/source/api/modelscope.models.rst
  29. +7
    -0
      docs/source/api/modelscope.pipelines.audio.rst
  30. +26
    -0
      docs/source/api/modelscope.pipelines.cv.rst
  31. +18
    -0
      docs/source/api/modelscope.pipelines.multi_modal.rst
  32. +26
    -0
      docs/source/api/modelscope.pipelines.nlp.rst
  33. +53
    -0
      docs/source/api/modelscope.pipelines.rst
  34. +50
    -0
      docs/source/api/modelscope.preprocessors.rst
  35. +18
    -0
      docs/source/api/modelscope.pydatasets.rst
  36. +32
    -0
      docs/source/api/modelscope.rst
  37. +5
    -5
      docs/source/api/modelscope.trainers.nlp.rst
  38. +7
    -9
      docs/source/api/modelscope.trainers.rst
  39. +66
    -0
      docs/source/api/modelscope.utils.rst
  40. +3
    -3
      docs/source/api/modules.rst
  41. +5
    -5
      docs/source/conf.py
  42. +8
    -8
      docs/source/index.rst
  43. +16
    -16
      docs/source/quick_start.md
  44. +4
    -4
      docs/source/tutorials/pipeline.md
  45. +0
    -48
      maas_lib/pipelines/default.py
  46. +0
    -0
      modelscope/__init__.py
  47. +0
    -0
      modelscope/fileio/__init__.py
  48. +0
    -0
      modelscope/fileio/file.py
  49. +0
    -0
      modelscope/fileio/format/__init__.py
  50. +0
    -0
      modelscope/fileio/format/base.py
  51. +0
    -0
      modelscope/fileio/format/json.py
  52. +0
    -0
      modelscope/fileio/format/yaml.py
  53. +0
    -0
      modelscope/fileio/io.py
  54. +0
    -0
      modelscope/models/__init__.py
  55. +4
    -4
      modelscope/models/base.py
  56. +2
    -2
      modelscope/models/builder.py
  57. +0
    -0
      modelscope/models/cv/__init__.py
  58. +0
    -0
      modelscope/models/cv/cartoon/__init__.py
  59. +0
    -0
      modelscope/models/cv/cartoon/facelib/LICENSE
  60. +0
    -0
      modelscope/models/cv/cartoon/facelib/LK/__init__.py
  61. +0
    -0
      modelscope/models/cv/cartoon/facelib/LK/lk.py
  62. +0
    -0
      modelscope/models/cv/cartoon/facelib/__init__.py
  63. +0
    -0
      modelscope/models/cv/cartoon/facelib/config.py
  64. +0
    -0
      modelscope/models/cv/cartoon/facelib/face_detector.py
  65. +0
    -0
      modelscope/models/cv/cartoon/facelib/face_landmark.py
  66. +0
    -0
      modelscope/models/cv/cartoon/facelib/facer.py
  67. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/LICENSE
  68. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/README.md
  69. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/__init__.py
  70. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/__init__.py
  71. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py
  72. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py
  73. +0
    -0
      modelscope/models/cv/cartoon/utils.py
  74. +0
    -0
      modelscope/models/nlp/__init__.py
  75. +1
    -1
      modelscope/models/nlp/sequence_classification_model.py
  76. +1
    -1
      modelscope/models/nlp/text_generation_model.py
  77. +0
    -0
      modelscope/pipelines/__init__.py
  78. +0
    -0
      modelscope/pipelines/audio/__init__.py
  79. +6
    -6
      modelscope/pipelines/base.py
  80. +54
    -5
      modelscope/pipelines/builder.py
  81. +0
    -0
      modelscope/pipelines/cv/__init__.py
  82. +7
    -7
      modelscope/pipelines/cv/image_cartoon_pipeline.py
  83. +4
    -4
      modelscope/pipelines/cv/image_matting_pipeline.py
  84. +0
    -0
      modelscope/pipelines/multi_modal/__init__.py
  85. +4
    -4
      modelscope/pipelines/multi_modal/image_captioning.py
  86. +0
    -0
      modelscope/pipelines/nlp/__init__.py
  87. +3
    -3
      modelscope/pipelines/nlp/sequence_classification_pipeline.py
  88. +4
    -4
      modelscope/pipelines/nlp/text_generation_pipeline.py
  89. +1
    -1
      modelscope/pipelines/util.py
  90. +0
    -0
      modelscope/preprocessors/__init__.py
  91. +0
    -0
      modelscope/preprocessors/base.py
  92. +3
    -3
      modelscope/preprocessors/builder.py
  93. +0
    -0
      modelscope/preprocessors/common.py
  94. +2
    -2
      modelscope/preprocessors/image.py
  95. +2
    -2
      modelscope/preprocessors/nlp.py
  96. +0
    -0
      modelscope/pydatasets/__init__.py
  97. +2
    -2
      modelscope/pydatasets/py_dataset.py
  98. +1
    -1
      modelscope/tools/eval.py
  99. +1
    -1
      modelscope/tools/train.py
  100. +0
    -0
      modelscope/trainers/__init__.py

+ 1
- 1
.dev_scripts/build_docs.sh View File

@@ -4,5 +4,5 @@ rm -rf build


# update api rst # update api rst
#rm -rf source/api/ #rm -rf source/api/
#sphinx-apidoc --module-first -o source/api/ ../maas_lib/
#sphinx-apidoc --module-first -o source/api/ ../modelscope/
make html make html

+ 3
- 3
.dev_scripts/linter.sh View File

@@ -1,3 +1,3 @@
yapf -r -i maas_lib/ configs/ tests/ setup.py
isort -rc maas_lib/ configs/ tests/ setup.py
flake8 maas_lib/ configs/ tests/ setup.py
yapf -r -i modelscope/ configs/ tests/ setup.py
isort -rc modelscope/ configs/ tests/ setup.py
flake8 modelscope/ configs/ tests/ setup.py

+ 2
- 2
LICENSE View File

@@ -1,4 +1,4 @@
Copyright 2022-2023 Alibaba MaaS. All rights reserved.
Copyright 2022-2023 Alibaba ModelScope. All rights reserved.


Apache License Apache License
Version 2.0, January 2004 Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2022-2023 Alibaba MaaS. All rights reserved.
same "printed page" as the copyright notice for easier same "printed page" as the copyright notice for easier
identification within third-party archives. identification within third-party archives.


Copyright 2020-2022 Alibaba MaaS.
Copyright 2020-2022 Alibaba ModelScope.


Licensed under the Apache License, Version 2.0 (the "License"); Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.


+ 1
- 1
MANIFEST.in View File

@@ -1 +1 @@
recursive-include maas_lib/configs *.py
recursive-include modelscope/configs *.py

+ 1
- 1
README.md View File

@@ -1,6 +1,6 @@
# Introduction # Introduction


MaaS library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba.
ModelScope library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba.


# Design doc # Design doc




+ 1
- 1
configs/README.md View File

@@ -1 +1 @@
This folder will host example configs for each model supported by maas_lib.
This folder will host example configs for each model supported by modelscope.

+ 0
- 34
docs/source/api/maas_lib.fileio.format.rst View File

@@ -1,34 +0,0 @@
maas\_lib.fileio.format package
===============================

.. automodule:: maas_lib.fileio.format
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.fileio.format.base module
-----------------------------------

.. automodule:: maas_lib.fileio.format.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.format.json module
-----------------------------------

.. automodule:: maas_lib.fileio.format.json
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.format.yaml module
-----------------------------------

.. automodule:: maas_lib.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.fileio.rst View File

@@ -1,34 +0,0 @@
maas\_lib.fileio package
========================

.. automodule:: maas_lib.fileio
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.fileio.format

Submodules
----------

maas\_lib.fileio.file module
----------------------------

.. automodule:: maas_lib.fileio.file
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.io module
--------------------------

.. automodule:: maas_lib.fileio.io
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.models.nlp.rst View File

@@ -1,18 +0,0 @@
maas\_lib.models.nlp package
============================

.. automodule:: maas_lib.models.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.models.nlp.sequence\_classification\_model module
-----------------------------------------------------------

.. automodule:: maas_lib.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.models.rst View File

@@ -1,34 +0,0 @@
maas\_lib.models package
========================

.. automodule:: maas_lib.models
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.models.nlp

Submodules
----------

maas\_lib.models.base module
----------------------------

.. automodule:: maas_lib.models.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.models.builder module
-------------------------------

.. automodule:: maas_lib.models.builder
:members:
:undoc-members:
:show-inheritance:

+ 0
- 7
docs/source/api/maas_lib.pipelines.audio.rst View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.audio package
=================================

.. automodule:: maas_lib.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.pipelines.cv.rst View File

@@ -1,18 +0,0 @@
maas\_lib.pipelines.cv package
==============================

.. automodule:: maas_lib.pipelines.cv
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.pipelines.cv.image\_matting module
--------------------------------------------

.. automodule:: maas_lib.pipelines.cv.image_matting
:members:
:undoc-members:
:show-inheritance:

+ 0
- 7
docs/source/api/maas_lib.pipelines.multi_modal.rst View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.multi\_modal package
========================================

.. automodule:: maas_lib.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:

+ 0
- 50
docs/source/api/maas_lib.preprocessors.rst View File

@@ -1,50 +0,0 @@
maas\_lib.preprocessors package
===============================

.. automodule:: maas_lib.preprocessors
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.preprocessors.base module
-----------------------------------

.. automodule:: maas_lib.preprocessors.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.builder module
--------------------------------------

.. automodule:: maas_lib.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.common module
-------------------------------------

.. automodule:: maas_lib.preprocessors.common
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.image module
------------------------------------

.. automodule:: maas_lib.preprocessors.image
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.nlp module
----------------------------------

.. automodule:: maas_lib.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

+ 0
- 30
docs/source/api/maas_lib.rst View File

@@ -1,30 +0,0 @@
maas\_lib package
=================

.. automodule:: maas_lib
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.fileio
maas_lib.models
maas_lib.pipelines
maas_lib.preprocessors
maas_lib.utils

Submodules
----------

maas\_lib.version module
------------------------

.. automodule:: maas_lib.version
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.trainers.nlp.rst View File

@@ -1,18 +0,0 @@
maas\_lib.trainers.nlp package
==============================

.. automodule:: maas_lib.trainers.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.trainers.nlp.sequence\_classification\_trainer module
---------------------------------------------------------------

.. automodule:: maas_lib.trainers.nlp.sequence_classification_trainer
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.trainers.rst View File

@@ -1,34 +0,0 @@
maas\_lib.trainers package
==========================

.. automodule:: maas_lib.trainers
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.trainers.nlp

Submodules
----------

maas\_lib.trainers.base module
------------------------------

.. automodule:: maas_lib.trainers.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.trainers.builder module
---------------------------------

.. automodule:: maas_lib.trainers.builder
:members:
:undoc-members:
:show-inheritance:

+ 0
- 58
docs/source/api/maas_lib.utils.rst View File

@@ -1,58 +0,0 @@
maas\_lib.utils package
=======================

.. automodule:: maas_lib.utils
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.utils.config module
-----------------------------

.. automodule:: maas_lib.utils.config
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.constant module
-------------------------------

.. automodule:: maas_lib.utils.constant
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.logger module
-----------------------------

.. automodule:: maas_lib.utils.logger
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.pymod module
----------------------------

.. automodule:: maas_lib.utils.pymod
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.registry module
-------------------------------

.. automodule:: maas_lib.utils.registry
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.type\_assert module
-----------------------------------

.. automodule:: maas_lib.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

+ 34
- 0
docs/source/api/modelscope.fileio.format.rst View File

@@ -0,0 +1,34 @@
modelscope.fileio.format package
================================

.. automodule:: modelscope.fileio.format
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.fileio.format.base module
------------------------------------

.. automodule:: modelscope.fileio.format.base
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.format.json module
------------------------------------

.. automodule:: modelscope.fileio.format.json
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.format.yaml module
------------------------------------

.. automodule:: modelscope.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

+ 34
- 0
docs/source/api/modelscope.fileio.rst View File

@@ -0,0 +1,34 @@
modelscope.fileio package
=========================

.. automodule:: modelscope.fileio
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.fileio.format

Submodules
----------

modelscope.fileio.file module
-----------------------------

.. automodule:: modelscope.fileio.file
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.io module
---------------------------

.. automodule:: modelscope.fileio.io
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.models.cv.cartoon.facelib.LK.rst View File

@@ -0,0 +1,18 @@
modelscope.models.cv.cartoon.facelib.LK package
===============================================

.. automodule:: modelscope.models.cv.cartoon.facelib.LK
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.cv.cartoon.facelib.LK.lk module
-------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.LK.lk
:members:
:undoc-members:
:show-inheritance:

+ 50
- 0
docs/source/api/modelscope.models.cv.cartoon.facelib.rst View File

@@ -0,0 +1,50 @@
modelscope.models.cv.cartoon.facelib package
============================================

.. automodule:: modelscope.models.cv.cartoon.facelib
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.facelib.LK

Submodules
----------

modelscope.models.cv.cartoon.facelib.config module
--------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.config
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.face\_detector module
----------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.face_detector
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.face\_landmark module
----------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.face_landmark
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.facer module
-------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.facer
:members:
:undoc-members:
:show-inheritance:

+ 15
- 0
docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.rst View File

@@ -0,0 +1,15 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch package
===================================================

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.mtcnn_pytorch.src

+ 26
- 0
docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.src.rst View File

@@ -0,0 +1,26 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch.src package
=======================================================

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.cv.cartoon.mtcnn\_pytorch.src.align\_trans module
-------------------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.mtcnn\_pytorch.src.matlab\_cp2tform module
-----------------------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.matlab_cp2tform
:members:
:undoc-members:
:show-inheritance:

+ 27
- 0
docs/source/api/modelscope.models.cv.cartoon.rst View File

@@ -0,0 +1,27 @@
modelscope.models.cv.cartoon package
====================================

.. automodule:: modelscope.models.cv.cartoon
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.facelib
modelscope.models.cv.cartoon.mtcnn_pytorch

Submodules
----------

modelscope.models.cv.cartoon.utils module
-----------------------------------------

.. automodule:: modelscope.models.cv.cartoon.utils
:members:
:undoc-members:
:show-inheritance:

+ 15
- 0
docs/source/api/modelscope.models.cv.rst View File

@@ -0,0 +1,15 @@
modelscope.models.cv package
============================

.. automodule:: modelscope.models.cv
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon

+ 26
- 0
docs/source/api/modelscope.models.nlp.rst View File

@@ -0,0 +1,26 @@
modelscope.models.nlp package
=============================

.. automodule:: modelscope.models.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.nlp.sequence\_classification\_model module
------------------------------------------------------------

.. automodule:: modelscope.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:

modelscope.models.nlp.text\_generation\_model module
----------------------------------------------------

.. automodule:: modelscope.models.nlp.text_generation_model
:members:
:undoc-members:
:show-inheritance:

+ 35
- 0
docs/source/api/modelscope.models.rst View File

@@ -0,0 +1,35 @@
modelscope.models package
=========================

.. automodule:: modelscope.models
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv
modelscope.models.nlp

Submodules
----------

modelscope.models.base module
-----------------------------

.. automodule:: modelscope.models.base
:members:
:undoc-members:
:show-inheritance:

modelscope.models.builder module
--------------------------------

.. automodule:: modelscope.models.builder
:members:
:undoc-members:
:show-inheritance:

+ 7
- 0
docs/source/api/modelscope.pipelines.audio.rst View File

@@ -0,0 +1,7 @@
modelscope.pipelines.audio package
==================================

.. automodule:: modelscope.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

+ 26
- 0
docs/source/api/modelscope.pipelines.cv.rst View File

@@ -0,0 +1,26 @@
modelscope.pipelines.cv package
===============================

.. automodule:: modelscope.pipelines.cv
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.cv.image\_cartoon\_pipeline module
-------------------------------------------------------

.. automodule:: modelscope.pipelines.cv.image_cartoon_pipeline
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.cv.image\_matting\_pipeline module
-------------------------------------------------------

.. automodule:: modelscope.pipelines.cv.image_matting_pipeline
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.pipelines.multi_modal.rst View File

@@ -0,0 +1,18 @@
modelscope.pipelines.multi\_modal package
=========================================

.. automodule:: modelscope.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.multi\_modal.image\_captioning module
----------------------------------------------------------

.. automodule:: modelscope.pipelines.multi_modal.image_captioning
:members:
:undoc-members:
:show-inheritance:

+ 26
- 0
docs/source/api/modelscope.pipelines.nlp.rst View File

@@ -0,0 +1,26 @@
modelscope.pipelines.nlp package
================================

.. automodule:: modelscope.pipelines.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.nlp.sequence\_classification\_pipeline module
------------------------------------------------------------------

.. automodule:: modelscope.pipelines.nlp.sequence_classification_pipeline
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.nlp.text\_generation\_pipeline module
----------------------------------------------------------

.. automodule:: modelscope.pipelines.nlp.text_generation_pipeline
:members:
:undoc-members:
:show-inheritance:

+ 53
- 0
docs/source/api/modelscope.pipelines.rst View File

@@ -0,0 +1,53 @@
modelscope.pipelines package
============================

.. automodule:: modelscope.pipelines
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.pipelines.audio
modelscope.pipelines.cv
modelscope.pipelines.multi_modal
modelscope.pipelines.nlp

Submodules
----------

modelscope.pipelines.base module
--------------------------------

.. automodule:: modelscope.pipelines.base
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.builder module
-----------------------------------

.. automodule:: modelscope.pipelines.builder
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.default module
-----------------------------------

.. automodule:: modelscope.pipelines.default
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.util module
--------------------------------

.. automodule:: modelscope.pipelines.util
:members:
:undoc-members:
:show-inheritance:

+ 50
- 0
docs/source/api/modelscope.preprocessors.rst View File

@@ -0,0 +1,50 @@
modelscope.preprocessors package
================================

.. automodule:: modelscope.preprocessors
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.preprocessors.base module
------------------------------------

.. automodule:: modelscope.preprocessors.base
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.builder module
---------------------------------------

.. automodule:: modelscope.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.common module
--------------------------------------

.. automodule:: modelscope.preprocessors.common
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.image module
-------------------------------------

.. automodule:: modelscope.preprocessors.image
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.nlp module
-----------------------------------

.. automodule:: modelscope.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.pydatasets.rst View File

@@ -0,0 +1,18 @@
modelscope.pydatasets package
=============================

.. automodule:: modelscope.pydatasets
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pydatasets.py\_dataset module
----------------------------------------

.. automodule:: modelscope.pydatasets.py_dataset
:members:
:undoc-members:
:show-inheritance:

+ 32
- 0
docs/source/api/modelscope.rst View File

@@ -0,0 +1,32 @@
modelscope package
==================

.. automodule:: modelscope
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.fileio
modelscope.models
modelscope.pipelines
modelscope.preprocessors
modelscope.pydatasets
modelscope.trainers
modelscope.utils

Submodules
----------

modelscope.version module
-------------------------

.. automodule:: modelscope.version
:members:
:undoc-members:
:show-inheritance:

docs/source/api/maas_lib.pipelines.nlp.rst → docs/source/api/modelscope.trainers.nlp.rst View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines.nlp package
modelscope.trainers.nlp package
=============================== ===============================


.. automodule:: maas_lib.pipelines.nlp
.. automodule:: modelscope.trainers.nlp
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:
@@ -9,10 +9,10 @@ maas\_lib.pipelines.nlp package
Submodules Submodules
---------- ----------


maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module
-----------------------------------------------------------------
modelscope.trainers.nlp.sequence\_classification\_trainer module
----------------------------------------------------------------


.. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline
.. automodule:: modelscope.trainers.nlp.sequence_classification_trainer
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:

docs/source/api/maas_lib.pipelines.rst → docs/source/api/modelscope.trainers.rst View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines package
modelscope.trainers package
=========================== ===========================


.. automodule:: maas_lib.pipelines
.. automodule:: modelscope.trainers
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:
@@ -12,25 +12,23 @@ Subpackages
.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4


maas_lib.pipelines.cv
maas_lib.pipelines.multi_modal
maas_lib.pipelines.nlp
modelscope.trainers.nlp


Submodules Submodules
---------- ----------


maas\_lib.pipelines.base module
modelscope.trainers.base module
------------------------------- -------------------------------


.. automodule:: maas_lib.pipelines.base
.. automodule:: modelscope.trainers.base
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:


maas\_lib.pipelines.builder module
modelscope.trainers.builder module
---------------------------------- ----------------------------------


.. automodule:: maas_lib.pipelines.builder
.. automodule:: modelscope.trainers.builder
:members: :members:
:undoc-members: :undoc-members:
:show-inheritance: :show-inheritance:

+ 66
- 0
docs/source/api/modelscope.utils.rst View File

@@ -0,0 +1,66 @@
modelscope.utils package
========================

.. automodule:: modelscope.utils
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.utils.config module
------------------------------

.. automodule:: modelscope.utils.config
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.constant module
--------------------------------

.. automodule:: modelscope.utils.constant
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.hub module
---------------------------

.. automodule:: modelscope.utils.hub
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.logger module
------------------------------

.. automodule:: modelscope.utils.logger
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.pymod module
-----------------------------

.. automodule:: modelscope.utils.pymod
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.registry module
--------------------------------

.. automodule:: modelscope.utils.registry
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.type\_assert module
------------------------------------

.. automodule:: modelscope.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

+ 3
- 3
docs/source/api/modules.rst View File

@@ -1,7 +1,7 @@
maas_lib
========
modelscope
==========


.. toctree:: .. toctree::
:maxdepth: 4 :maxdepth: 4


maas_lib
modelscope

+ 5
- 5
docs/source/conf.py View File

@@ -18,10 +18,10 @@ import sphinx_rtd_theme
sys.path.insert(0, os.path.abspath('../../')) sys.path.insert(0, os.path.abspath('../../'))
# -- Project information ----------------------------------------------------- # -- Project information -----------------------------------------------------


project = 'maas_lib'
copyright = '2022-2023, Alibaba MaaS'
author = 'maas_lib Authors'
version_file = '../../maas_lib/version.py'
project = 'modelscope'
copyright = '2022-2023, Alibaba ModelScope'
author = 'modelscope Authors'
version_file = '../../modelscope/version.py'




def get_version(): def get_version():
@@ -88,7 +88,7 @@ html_static_path = ['_static']


# -- Options for HTMLHelp output --------------------------------------------- # -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder. # Output file base name for HTML help builder.
htmlhelp_basename = 'maas_lib_doc'
htmlhelp_basename = 'modelscope_doc'


# -- Extension configuration ------------------------------------------------- # -- Extension configuration -------------------------------------------------
# Ignore >>> when copying code # Ignore >>> when copying code


+ 8
- 8
docs/source/index.rst View File

@@ -1,11 +1,11 @@
.. maas_lib documentation file,
.. modelscope documentation file,
You can adapt this file completely to your liking, but it should at least You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive. contain the root `toctree` directive.


MaasLib DOCUMENTATION
ModelScope DOCUMENTATION
======================================= =======================================


MaasLib doc
ModelScope doc


.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
@@ -30,11 +30,11 @@ MaasLib doc
:maxdepth: 10 :maxdepth: 10
:caption: API Doc :caption: API Doc


api/maas_lib.preprocessors
api/maas_lib.models
api/maas_lib.pipelines
api/maas_lib.fileio
api/maas_lib.utils
api/modelscope.preprocessors
api/modelscope.models
api/modelscope.pipelines
api/modelscope.fileio
api/modelscope.utils




Indices and tables Indices and tables


+ 16
- 16
docs/source/quick_start.md View File

@@ -5,39 +5,39 @@


安装完成后,执行如下命令为maas library创建对应的python环境。 安装完成后,执行如下命令为maas library创建对应的python环境。
```shell ```shell
conda create -n maas python=3.6
conda activate maas
conda create -n modelscope python=3.6
conda activate modelscope
``` ```
检查python和pip命令是否切换到conda环境下。 检查python和pip命令是否切换到conda环境下。
```shell ```shell
which python which python
# ~/workspace/anaconda3/envs/maas/bin/python
# ~/workspace/anaconda3/envs/modelscope/bin/python


which pip which pip
# ~/workspace/anaconda3/envs/maas/bin/pip
# ~/workspace/anaconda3/envs/modelscope/bin/pip
``` ```
注: 本项目只支持`python3`环境,请勿使用python2环境。 注: 本项目只支持`python3`环境,请勿使用python2环境。


## 第三方依赖安装 ## 第三方依赖安装


MaaS Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架:
ModelScope Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架:


* [Pytorch安装指导](https://pytorch.org/get-started/locally/) * [Pytorch安装指导](https://pytorch.org/get-started/locally/)
* [Tensorflow安装指导](https://www.tensorflow.org/install/pip) * [Tensorflow安装指导](https://www.tensorflow.org/install/pip)




## MaaS library 安装
## ModelScope library 安装


注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。 注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。


### pip安装 ### pip安装
```shell ```shell
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/maas.txt
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/modelscope.txt
``` ```


安装成功后,可以执行如下命令进行验证安装是否正确 安装成功后,可以执行如下命令进行验证安装是否正确
```shell ```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
``` ```




@@ -45,11 +45,11 @@ python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting


适合本地开发调试使用,修改源码后可以直接执行 适合本地开发调试使用,修改源码后可以直接执行
```shell ```shell
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git maaslib
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git modelscope
git fetch origin master git fetch origin master
git checkout master git checkout master


cd maaslib
cd modelscope


#安装依赖 #安装依赖
pip install -r requirements.txt pip install -r requirements.txt
@@ -60,7 +60,7 @@ export PYTHONPATH=`pwd`


安装成功后,可以执行如下命令进行验证安装是否正确 安装成功后,可以执行如下命令进行验证安装是否正确
```shell ```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
``` ```




@@ -79,8 +79,8 @@ pipeline函数提供了简洁的推理接口,示例如下, 更多pipeline介
```python ```python
import cv2 import cv2
import os.path as osp import os.path as osp
from maas_lib.pipelines import pipeline
from maas_lib.utils.constant import Tasks
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks


# 根据任务名创建pipeline # 根据任务名创建pipeline
img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person') img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person')
@@ -99,9 +99,9 @@ print(f'Output written to {osp.abspath("result.png")}')
```python ```python
import cv2 import cv2
import os.path as osp import os.path as osp
from maas_lib.pipelines import pipeline
from maas_lib.utils.constant import Tasks
from maas_lib.pydatasets import PyDataset
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks
from modelscope.pydatasets import PyDataset


# 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹 # 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹
input_location = [ input_location = [


+ 4
- 4
docs/source/tutorials/pipeline.md View File

@@ -19,7 +19,7 @@
1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象 1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象
执行如下python代码 执行如下python代码
```python ```python
>>> from maas_lib.pipelines import pipeline
>>> from modelscope.pipelines import pipeline
>>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person') >>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person')
``` ```


@@ -65,8 +65,8 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz


创建tokenizer和模型 创建tokenizer和模型
```python ```python
>>> from maas_lib.models import Model
>>> from maas_lib.preprocessors import SequenceClassificationPreprocessor
>>> from modelscope.models import Model
>>> from modelscope.preprocessors import SequenceClassificationPreprocessor
>>> model = Model.from_pretrained('damo/bert-base-sst2') >>> model = Model.from_pretrained('damo/bert-base-sst2')
>>> tokenizer = SequenceClassificationPreprocessor( >>> tokenizer = SequenceClassificationPreprocessor(
model.model_dir, first_sequence='sentence', second_sequence=None) model.model_dir, first_sequence='sentence', second_sequence=None)
@@ -74,7 +74,7 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz


使用tokenizer和模型对象创建pipeline 使用tokenizer和模型对象创建pipeline
```python ```python
>>> from maas_lib.pipelines import pipeline
>>> from modelscope.pipelines import pipeline
>>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer) >>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer)
>>> semantic_cls("Hello world!") >>> semantic_cls("Hello world!")
``` ```


+ 0
- 48
maas_lib/pipelines/default.py View File

@@ -1,48 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.

from maas_lib.utils.constant import Tasks

DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
}


def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.

Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'

DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)


def get_default_pipeline_info(task):
""" Get default info for certain task.

Args:
task (str): task name.

Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""
assert task in DEFAULT_MODEL_FOR_PIPELINE, \
f'No default pipeline is registered for Task {task}'

pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

maas_lib/__init__.py → modelscope/__init__.py View File


maas_lib/fileio/__init__.py → modelscope/fileio/__init__.py View File


maas_lib/fileio/file.py → modelscope/fileio/file.py View File


maas_lib/fileio/format/__init__.py → modelscope/fileio/format/__init__.py View File


maas_lib/fileio/format/base.py → modelscope/fileio/format/base.py View File


maas_lib/fileio/format/json.py → modelscope/fileio/format/json.py View File


maas_lib/fileio/format/yaml.py → modelscope/fileio/format/yaml.py View File


maas_lib/fileio/io.py → modelscope/fileio/io.py View File


maas_lib/models/__init__.py → modelscope/models/__init__.py View File


maas_lib/models/base.py → modelscope/models/base.py View File

@@ -7,10 +7,10 @@ from typing import Dict, List, Tuple, Union
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download
from maas_hub.snapshot_download import snapshot_download from maas_hub.snapshot_download import snapshot_download


from maas_lib.models.builder import build_model
from maas_lib.utils.config import Config
from maas_lib.utils.constant import CONFIGFILE
from maas_lib.utils.hub import get_model_cache_dir
from modelscope.models.builder import build_model
from modelscope.utils.config import Config
from modelscope.utils.constant import CONFIGFILE
from modelscope.utils.hub import get_model_cache_dir


Tensor = Union['torch.Tensor', 'tf.Tensor'] Tensor = Union['torch.Tensor', 'tf.Tensor']



maas_lib/models/builder.py → modelscope/models/builder.py View File

@@ -1,7 +1,7 @@
# Copyright (c) Alibaba, Inc. and its affiliates. # Copyright (c) Alibaba, Inc. and its affiliates.


from maas_lib.utils.config import ConfigDict
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.utils.config import ConfigDict
from modelscope.utils.registry import Registry, build_from_cfg


MODELS = Registry('models') MODELS = Registry('models')



maas_lib/models/cv/__init__.py → modelscope/models/cv/__init__.py View File


maas_lib/models/cv/cartoon/__init__.py → modelscope/models/cv/cartoon/__init__.py View File


maas_lib/models/cv/cartoon/facelib/LICENSE → modelscope/models/cv/cartoon/facelib/LICENSE View File


maas_lib/models/cv/cartoon/facelib/LK/__init__.py → modelscope/models/cv/cartoon/facelib/LK/__init__.py View File


maas_lib/models/cv/cartoon/facelib/LK/lk.py → modelscope/models/cv/cartoon/facelib/LK/lk.py View File


maas_lib/models/cv/cartoon/facelib/__init__.py → modelscope/models/cv/cartoon/facelib/__init__.py View File


maas_lib/models/cv/cartoon/facelib/config.py → modelscope/models/cv/cartoon/facelib/config.py View File


maas_lib/models/cv/cartoon/facelib/face_detector.py → modelscope/models/cv/cartoon/facelib/face_detector.py View File


maas_lib/models/cv/cartoon/facelib/face_landmark.py → modelscope/models/cv/cartoon/facelib/face_landmark.py View File


maas_lib/models/cv/cartoon/facelib/facer.py → modelscope/models/cv/cartoon/facelib/facer.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/LICENSE → modelscope/models/cv/cartoon/mtcnn_pytorch/LICENSE View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/README.md → modelscope/models/cv/cartoon/mtcnn_pytorch/README.md View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/__init__.py → modelscope/models/cv/cartoon/mtcnn_pytorch/__init__.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/__init__.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/__init__.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py View File


maas_lib/models/cv/cartoon/utils.py → modelscope/models/cv/cartoon/utils.py View File


maas_lib/models/nlp/__init__.py → modelscope/models/nlp/__init__.py View File


maas_lib/models/nlp/sequence_classification_model.py → modelscope/models/nlp/sequence_classification_model.py View File

@@ -2,7 +2,7 @@ from typing import Any, Dict


import numpy as np import numpy as np


from maas_lib.utils.constant import Tasks
from modelscope.utils.constant import Tasks
from ..base import Model from ..base import Model
from ..builder import MODELS from ..builder import MODELS



maas_lib/models/nlp/text_generation_model.py → modelscope/models/nlp/text_generation_model.py View File

@@ -1,6 +1,6 @@
from typing import Any, Dict from typing import Any, Dict


from maas_lib.utils.constant import Tasks
from modelscope.utils.constant import Tasks
from ..base import Model, Tensor from ..base import Model, Tensor
from ..builder import MODELS from ..builder import MODELS



maas_lib/pipelines/__init__.py → modelscope/pipelines/__init__.py View File


maas_lib/pipelines/audio/__init__.py → modelscope/pipelines/audio/__init__.py View File


maas_lib/pipelines/base.py → modelscope/pipelines/base.py View File

@@ -6,11 +6,11 @@ from typing import Any, Dict, Generator, List, Union


from maas_hub.snapshot_download import snapshot_download from maas_hub.snapshot_download import snapshot_download


from maas_lib.models.base import Model
from maas_lib.preprocessors import Preprocessor
from maas_lib.pydatasets import PyDataset
from maas_lib.utils.config import Config
from maas_lib.utils.hub import get_model_cache_dir
from modelscope.models.base import Model
from modelscope.preprocessors import Preprocessor
from modelscope.pydatasets import PyDataset
from modelscope.utils.config import Config
from modelscope.utils.hub import get_model_cache_dir
from .util import is_model_name from .util import is_model_name


Tensor = Union['torch.Tensor', 'tf.Tensor'] Tensor = Union['torch.Tensor', 'tf.Tensor']
@@ -75,7 +75,7 @@ class Pipeline(ABC):
def __call__(self, input: Union[Input, List[Input]], *args, def __call__(self, input: Union[Input, List[Input]], *args,
**post_kwargs) -> Union[Dict[str, Any], Generator]: **post_kwargs) -> Union[Dict[str, Any], Generator]:
# model provider should leave it as it is # model provider should leave it as it is
# maas library developer will handle this function
# modelscope library developer will handle this function


# simple showcase, need to support iterator type for both tensorflow and pytorch # simple showcase, need to support iterator type for both tensorflow and pytorch
# input_dict = self._handle_input(input) # input_dict = self._handle_input(input)

maas_lib/pipelines/builder.py → modelscope/pipelines/builder.py View File

@@ -6,16 +6,26 @@ from typing import Union
import json import json
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download


from maas_lib.models.base import Model
from maas_lib.utils.config import Config, ConfigDict
from maas_lib.utils.constant import CONFIGFILE, Tasks
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.models.base import Model
from modelscope.utils.config import Config, ConfigDict
from modelscope.utils.constant import CONFIGFILE, Tasks
from modelscope.utils.registry import Registry, build_from_cfg
from .base import Pipeline from .base import Pipeline
from .default import DEFAULT_MODEL_FOR_PIPELINE, get_default_pipeline_info
from .util import is_model_name from .util import is_model_name


PIPELINES = Registry('pipelines') PIPELINES = Registry('pipelines')


DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
Tasks.image_generation:
('cv_unet_person-image-cartoon', 'damo/cv_unet_image-matting_damo'),
}



def build_pipeline(cfg: ConfigDict, def build_pipeline(cfg: ConfigDict,
task_name: str = None, task_name: str = None,
@@ -84,3 +94,42 @@ def pipeline(task: str = None,
cfg.preprocessor = preprocessor cfg.preprocessor = preprocessor


return build_pipeline(cfg, task_name=task) return build_pipeline(cfg, task_name=task)


def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.

Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'

DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)


def get_default_pipeline_info(task):
""" Get default info for certain task.

Args:
task (str): task name.

Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""

if task not in DEFAULT_MODEL_FOR_PIPELINE:
# support pipeline which does not register default model
pipeline_name = list(PIPELINES.modules[task].keys())[0]
default_model = None
else:
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

maas_lib/pipelines/cv/__init__.py → modelscope/pipelines/cv/__init__.py View File


maas_lib/pipelines/cv/image_cartoon_pipeline.py → modelscope/pipelines/cv/image_cartoon_pipeline.py View File

@@ -6,14 +6,14 @@ import numpy as np
import PIL import PIL
import tensorflow as tf import tensorflow as tf


from maas_lib.models.cv.cartoon.facelib.facer import FaceAna
from maas_lib.models.cv.cartoon.mtcnn_pytorch.src.align_trans import (
from modelscope.models.cv.cartoon.facelib.facer import FaceAna
from modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans import (
get_reference_facial_points, warp_and_crop_face) get_reference_facial_points, warp_and_crop_face)
from maas_lib.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size
from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES



maas_lib/pipelines/cv/image_matting_pipeline.py → modelscope/pipelines/cv/image_matting_pipeline.py View File

@@ -5,10 +5,10 @@ import cv2
import numpy as np import numpy as np
import PIL import PIL


from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES



maas_lib/pipelines/multi_modal/__init__.py → modelscope/pipelines/multi_modal/__init__.py View File


maas_lib/pipelines/multi_modal/image_captioning.py → modelscope/pipelines/multi_modal/image_captioning.py View File

@@ -4,10 +4,10 @@ import numpy as np
import torch import torch
from PIL import Image from PIL import Image


from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline from ..base import Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES



maas_lib/pipelines/nlp/__init__.py → modelscope/pipelines/nlp/__init__.py View File


maas_lib/pipelines/nlp/sequence_classification_pipeline.py → modelscope/pipelines/nlp/sequence_classification_pipeline.py View File

@@ -5,9 +5,9 @@ from typing import Any, Dict, Union
import json import json
import numpy as np import numpy as np


from maas_lib.models.nlp import BertForSequenceClassification
from maas_lib.preprocessors import SequenceClassificationPreprocessor
from maas_lib.utils.constant import Tasks
from modelscope.models.nlp import BertForSequenceClassification
from modelscope.preprocessors import SequenceClassificationPreprocessor
from modelscope.utils.constant import Tasks
from ...models import Model from ...models import Model
from ..base import Input, Pipeline from ..base import Input, Pipeline
from ..builder import PIPELINES from ..builder import PIPELINES

maas_lib/pipelines/nlp/text_generation_pipeline.py → modelscope/pipelines/nlp/text_generation_pipeline.py View File

@@ -1,9 +1,9 @@
from typing import Dict, Optional, Union from typing import Dict, Optional, Union


from maas_lib.models import Model
from maas_lib.models.nlp import PalmForTextGenerationModel
from maas_lib.preprocessors import TextGenerationPreprocessor
from maas_lib.utils.constant import Tasks
from modelscope.models import Model
from modelscope.models.nlp import PalmForTextGenerationModel
from modelscope.preprocessors import TextGenerationPreprocessor
from modelscope.utils.constant import Tasks
from ..base import Pipeline, Tensor from ..base import Pipeline, Tensor
from ..builder import PIPELINES from ..builder import PIPELINES



maas_lib/pipelines/util.py → modelscope/pipelines/util.py View File

@@ -5,7 +5,7 @@ import os.path as osp
import json import json
from maas_hub.file_download import model_file_download from maas_hub.file_download import model_file_download


from maas_lib.utils.constant import CONFIGFILE
from modelscope.utils.constant import CONFIGFILE




def is_model_name(model): def is_model_name(model):

maas_lib/preprocessors/__init__.py → modelscope/preprocessors/__init__.py View File


maas_lib/preprocessors/base.py → modelscope/preprocessors/base.py View File


maas_lib/preprocessors/builder.py → modelscope/preprocessors/builder.py View File

@@ -1,8 +1,8 @@
# Copyright (c) Alibaba, Inc. and its affiliates. # Copyright (c) Alibaba, Inc. and its affiliates.


from maas_lib.utils.config import ConfigDict
from maas_lib.utils.constant import Fields
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.utils.config import ConfigDict
from modelscope.utils.constant import Fields
from modelscope.utils.registry import Registry, build_from_cfg


PREPROCESSORS = Registry('preprocessors') PREPROCESSORS = Registry('preprocessors')



maas_lib/preprocessors/common.py → modelscope/preprocessors/common.py View File


maas_lib/preprocessors/image.py → modelscope/preprocessors/image.py View File

@@ -4,8 +4,8 @@ from typing import Dict, Union


from PIL import Image, ImageOps from PIL import Image, ImageOps


from maas_lib.fileio import File
from maas_lib.utils.constant import Fields
from modelscope.fileio import File
from modelscope.utils.constant import Fields
from .builder import PREPROCESSORS from .builder import PREPROCESSORS





maas_lib/preprocessors/nlp.py → modelscope/preprocessors/nlp.py View File

@@ -5,8 +5,8 @@ from typing import Any, Dict, Union


from transformers import AutoTokenizer from transformers import AutoTokenizer


from maas_lib.utils.constant import Fields, InputFields
from maas_lib.utils.type_assert import type_assert
from modelscope.utils.constant import Fields, InputFields
from modelscope.utils.type_assert import type_assert
from .base import Preprocessor from .base import Preprocessor
from .builder import PREPROCESSORS from .builder import PREPROCESSORS



maas_lib/pydatasets/__init__.py → modelscope/pydatasets/__init__.py View File


maas_lib/pydatasets/py_dataset.py → modelscope/pydatasets/py_dataset.py View File

@@ -4,7 +4,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,


from datasets import Dataset, load_dataset from datasets import Dataset, load_dataset


from maas_lib.utils.logger import get_logger
from modelscope.utils.logger import get_logger


logger = get_logger() logger = get_logger()


@@ -52,7 +52,7 @@ class PyDataset:
Mapping[str, Union[str, Mapping[str, Union[str,
Sequence[str]]]]] = None Sequence[str]]]]] = None
) -> 'PyDataset': ) -> 'PyDataset':
"""Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset.
"""Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset.
Args: Args:


path (str): Path or name of the dataset. path (str): Path or name of the dataset.

maas_lib/tools/eval.py → modelscope/tools/eval.py View File

@@ -2,7 +2,7 @@


import argparse import argparse


from maas_lib.trainers import build_trainer
from modelscope.trainers import build_trainer




def parse_args(): def parse_args():

maas_lib/tools/train.py → modelscope/tools/train.py View File

@@ -2,7 +2,7 @@


import argparse import argparse


from maas_lib.trainers import build_trainer
from modelscope.trainers import build_trainer




def parse_args(): def parse_args():

maas_lib/trainers/__init__.py → modelscope/trainers/__init__.py View File


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save