Browse Source

[to #42373878] refactor maaslib to modelscope

1.  refactor maaslib to modelscope
2.  fix UT error
3.  support pipeline which does not register default model

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/8988388
master
wenmeng.zwm 3 years ago
parent
commit
1f6b376599
100 changed files with 735 additions and 502 deletions
  1. +1
    -1
      .dev_scripts/build_docs.sh
  2. +3
    -3
      .dev_scripts/linter.sh
  3. +2
    -2
      LICENSE
  4. +1
    -1
      MANIFEST.in
  5. +1
    -1
      README.md
  6. +1
    -1
      configs/README.md
  7. +0
    -34
      docs/source/api/maas_lib.fileio.format.rst
  8. +0
    -34
      docs/source/api/maas_lib.fileio.rst
  9. +0
    -18
      docs/source/api/maas_lib.models.nlp.rst
  10. +0
    -34
      docs/source/api/maas_lib.models.rst
  11. +0
    -7
      docs/source/api/maas_lib.pipelines.audio.rst
  12. +0
    -18
      docs/source/api/maas_lib.pipelines.cv.rst
  13. +0
    -7
      docs/source/api/maas_lib.pipelines.multi_modal.rst
  14. +0
    -50
      docs/source/api/maas_lib.preprocessors.rst
  15. +0
    -30
      docs/source/api/maas_lib.rst
  16. +0
    -18
      docs/source/api/maas_lib.trainers.nlp.rst
  17. +0
    -34
      docs/source/api/maas_lib.trainers.rst
  18. +0
    -58
      docs/source/api/maas_lib.utils.rst
  19. +34
    -0
      docs/source/api/modelscope.fileio.format.rst
  20. +34
    -0
      docs/source/api/modelscope.fileio.rst
  21. +18
    -0
      docs/source/api/modelscope.models.cv.cartoon.facelib.LK.rst
  22. +50
    -0
      docs/source/api/modelscope.models.cv.cartoon.facelib.rst
  23. +15
    -0
      docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.rst
  24. +26
    -0
      docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.src.rst
  25. +27
    -0
      docs/source/api/modelscope.models.cv.cartoon.rst
  26. +15
    -0
      docs/source/api/modelscope.models.cv.rst
  27. +26
    -0
      docs/source/api/modelscope.models.nlp.rst
  28. +35
    -0
      docs/source/api/modelscope.models.rst
  29. +7
    -0
      docs/source/api/modelscope.pipelines.audio.rst
  30. +26
    -0
      docs/source/api/modelscope.pipelines.cv.rst
  31. +18
    -0
      docs/source/api/modelscope.pipelines.multi_modal.rst
  32. +26
    -0
      docs/source/api/modelscope.pipelines.nlp.rst
  33. +53
    -0
      docs/source/api/modelscope.pipelines.rst
  34. +50
    -0
      docs/source/api/modelscope.preprocessors.rst
  35. +18
    -0
      docs/source/api/modelscope.pydatasets.rst
  36. +32
    -0
      docs/source/api/modelscope.rst
  37. +5
    -5
      docs/source/api/modelscope.trainers.nlp.rst
  38. +7
    -9
      docs/source/api/modelscope.trainers.rst
  39. +66
    -0
      docs/source/api/modelscope.utils.rst
  40. +3
    -3
      docs/source/api/modules.rst
  41. +5
    -5
      docs/source/conf.py
  42. +8
    -8
      docs/source/index.rst
  43. +16
    -16
      docs/source/quick_start.md
  44. +4
    -4
      docs/source/tutorials/pipeline.md
  45. +0
    -48
      maas_lib/pipelines/default.py
  46. +0
    -0
      modelscope/__init__.py
  47. +0
    -0
      modelscope/fileio/__init__.py
  48. +0
    -0
      modelscope/fileio/file.py
  49. +0
    -0
      modelscope/fileio/format/__init__.py
  50. +0
    -0
      modelscope/fileio/format/base.py
  51. +0
    -0
      modelscope/fileio/format/json.py
  52. +0
    -0
      modelscope/fileio/format/yaml.py
  53. +0
    -0
      modelscope/fileio/io.py
  54. +0
    -0
      modelscope/models/__init__.py
  55. +4
    -4
      modelscope/models/base.py
  56. +2
    -2
      modelscope/models/builder.py
  57. +0
    -0
      modelscope/models/cv/__init__.py
  58. +0
    -0
      modelscope/models/cv/cartoon/__init__.py
  59. +0
    -0
      modelscope/models/cv/cartoon/facelib/LICENSE
  60. +0
    -0
      modelscope/models/cv/cartoon/facelib/LK/__init__.py
  61. +0
    -0
      modelscope/models/cv/cartoon/facelib/LK/lk.py
  62. +0
    -0
      modelscope/models/cv/cartoon/facelib/__init__.py
  63. +0
    -0
      modelscope/models/cv/cartoon/facelib/config.py
  64. +0
    -0
      modelscope/models/cv/cartoon/facelib/face_detector.py
  65. +0
    -0
      modelscope/models/cv/cartoon/facelib/face_landmark.py
  66. +0
    -0
      modelscope/models/cv/cartoon/facelib/facer.py
  67. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/LICENSE
  68. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/README.md
  69. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/__init__.py
  70. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/__init__.py
  71. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py
  72. +0
    -0
      modelscope/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py
  73. +0
    -0
      modelscope/models/cv/cartoon/utils.py
  74. +0
    -0
      modelscope/models/nlp/__init__.py
  75. +1
    -1
      modelscope/models/nlp/sequence_classification_model.py
  76. +1
    -1
      modelscope/models/nlp/text_generation_model.py
  77. +0
    -0
      modelscope/pipelines/__init__.py
  78. +0
    -0
      modelscope/pipelines/audio/__init__.py
  79. +6
    -6
      modelscope/pipelines/base.py
  80. +54
    -5
      modelscope/pipelines/builder.py
  81. +0
    -0
      modelscope/pipelines/cv/__init__.py
  82. +7
    -7
      modelscope/pipelines/cv/image_cartoon_pipeline.py
  83. +4
    -4
      modelscope/pipelines/cv/image_matting_pipeline.py
  84. +0
    -0
      modelscope/pipelines/multi_modal/__init__.py
  85. +4
    -4
      modelscope/pipelines/multi_modal/image_captioning.py
  86. +0
    -0
      modelscope/pipelines/nlp/__init__.py
  87. +3
    -3
      modelscope/pipelines/nlp/sequence_classification_pipeline.py
  88. +4
    -4
      modelscope/pipelines/nlp/text_generation_pipeline.py
  89. +1
    -1
      modelscope/pipelines/util.py
  90. +0
    -0
      modelscope/preprocessors/__init__.py
  91. +0
    -0
      modelscope/preprocessors/base.py
  92. +3
    -3
      modelscope/preprocessors/builder.py
  93. +0
    -0
      modelscope/preprocessors/common.py
  94. +2
    -2
      modelscope/preprocessors/image.py
  95. +2
    -2
      modelscope/preprocessors/nlp.py
  96. +0
    -0
      modelscope/pydatasets/__init__.py
  97. +2
    -2
      modelscope/pydatasets/py_dataset.py
  98. +1
    -1
      modelscope/tools/eval.py
  99. +1
    -1
      modelscope/tools/train.py
  100. +0
    -0
      modelscope/trainers/__init__.py

+ 1
- 1
.dev_scripts/build_docs.sh View File

@@ -4,5 +4,5 @@ rm -rf build

# update api rst
#rm -rf source/api/
#sphinx-apidoc --module-first -o source/api/ ../maas_lib/
#sphinx-apidoc --module-first -o source/api/ ../modelscope/
make html

+ 3
- 3
.dev_scripts/linter.sh View File

@@ -1,3 +1,3 @@
yapf -r -i maas_lib/ configs/ tests/ setup.py
isort -rc maas_lib/ configs/ tests/ setup.py
flake8 maas_lib/ configs/ tests/ setup.py
yapf -r -i modelscope/ configs/ tests/ setup.py
isort -rc modelscope/ configs/ tests/ setup.py
flake8 modelscope/ configs/ tests/ setup.py

+ 2
- 2
LICENSE View File

@@ -1,4 +1,4 @@
Copyright 2022-2023 Alibaba MaaS. All rights reserved.
Copyright 2022-2023 Alibaba ModelScope. All rights reserved.

Apache License
Version 2.0, January 2004
@@ -188,7 +188,7 @@ Copyright 2022-2023 Alibaba MaaS. All rights reserved.
same "printed page" as the copyright notice for easier
identification within third-party archives.

Copyright 2020-2022 Alibaba MaaS.
Copyright 2020-2022 Alibaba ModelScope.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.


+ 1
- 1
MANIFEST.in View File

@@ -1 +1 @@
recursive-include maas_lib/configs *.py
recursive-include modelscope/configs *.py

+ 1
- 1
README.md View File

@@ -1,6 +1,6 @@
# Introduction

MaaS library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba.
ModelScope library is targeted to support training, evaluation and inference for the state of the art models provided by Mind and further support third-party models provided by users outside alibaba.

# Design doc



+ 1
- 1
configs/README.md View File

@@ -1 +1 @@
This folder will host example configs for each model supported by maas_lib.
This folder will host example configs for each model supported by modelscope.

+ 0
- 34
docs/source/api/maas_lib.fileio.format.rst View File

@@ -1,34 +0,0 @@
maas\_lib.fileio.format package
===============================

.. automodule:: maas_lib.fileio.format
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.fileio.format.base module
-----------------------------------

.. automodule:: maas_lib.fileio.format.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.format.json module
-----------------------------------

.. automodule:: maas_lib.fileio.format.json
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.format.yaml module
-----------------------------------

.. automodule:: maas_lib.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.fileio.rst View File

@@ -1,34 +0,0 @@
maas\_lib.fileio package
========================

.. automodule:: maas_lib.fileio
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.fileio.format

Submodules
----------

maas\_lib.fileio.file module
----------------------------

.. automodule:: maas_lib.fileio.file
:members:
:undoc-members:
:show-inheritance:

maas\_lib.fileio.io module
--------------------------

.. automodule:: maas_lib.fileio.io
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.models.nlp.rst View File

@@ -1,18 +0,0 @@
maas\_lib.models.nlp package
============================

.. automodule:: maas_lib.models.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.models.nlp.sequence\_classification\_model module
-----------------------------------------------------------

.. automodule:: maas_lib.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.models.rst View File

@@ -1,34 +0,0 @@
maas\_lib.models package
========================

.. automodule:: maas_lib.models
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.models.nlp

Submodules
----------

maas\_lib.models.base module
----------------------------

.. automodule:: maas_lib.models.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.models.builder module
-------------------------------

.. automodule:: maas_lib.models.builder
:members:
:undoc-members:
:show-inheritance:

+ 0
- 7
docs/source/api/maas_lib.pipelines.audio.rst View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.audio package
=================================

.. automodule:: maas_lib.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.pipelines.cv.rst View File

@@ -1,18 +0,0 @@
maas\_lib.pipelines.cv package
==============================

.. automodule:: maas_lib.pipelines.cv
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.pipelines.cv.image\_matting module
--------------------------------------------

.. automodule:: maas_lib.pipelines.cv.image_matting
:members:
:undoc-members:
:show-inheritance:

+ 0
- 7
docs/source/api/maas_lib.pipelines.multi_modal.rst View File

@@ -1,7 +0,0 @@
maas\_lib.pipelines.multi\_modal package
========================================

.. automodule:: maas_lib.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:

+ 0
- 50
docs/source/api/maas_lib.preprocessors.rst View File

@@ -1,50 +0,0 @@
maas\_lib.preprocessors package
===============================

.. automodule:: maas_lib.preprocessors
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.preprocessors.base module
-----------------------------------

.. automodule:: maas_lib.preprocessors.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.builder module
--------------------------------------

.. automodule:: maas_lib.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.common module
-------------------------------------

.. automodule:: maas_lib.preprocessors.common
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.image module
------------------------------------

.. automodule:: maas_lib.preprocessors.image
:members:
:undoc-members:
:show-inheritance:

maas\_lib.preprocessors.nlp module
----------------------------------

.. automodule:: maas_lib.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

+ 0
- 30
docs/source/api/maas_lib.rst View File

@@ -1,30 +0,0 @@
maas\_lib package
=================

.. automodule:: maas_lib
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.fileio
maas_lib.models
maas_lib.pipelines
maas_lib.preprocessors
maas_lib.utils

Submodules
----------

maas\_lib.version module
------------------------

.. automodule:: maas_lib.version
:members:
:undoc-members:
:show-inheritance:

+ 0
- 18
docs/source/api/maas_lib.trainers.nlp.rst View File

@@ -1,18 +0,0 @@
maas\_lib.trainers.nlp package
==============================

.. automodule:: maas_lib.trainers.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.trainers.nlp.sequence\_classification\_trainer module
---------------------------------------------------------------

.. automodule:: maas_lib.trainers.nlp.sequence_classification_trainer
:members:
:undoc-members:
:show-inheritance:

+ 0
- 34
docs/source/api/maas_lib.trainers.rst View File

@@ -1,34 +0,0 @@
maas\_lib.trainers package
==========================

.. automodule:: maas_lib.trainers
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

maas_lib.trainers.nlp

Submodules
----------

maas\_lib.trainers.base module
------------------------------

.. automodule:: maas_lib.trainers.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.trainers.builder module
---------------------------------

.. automodule:: maas_lib.trainers.builder
:members:
:undoc-members:
:show-inheritance:

+ 0
- 58
docs/source/api/maas_lib.utils.rst View File

@@ -1,58 +0,0 @@
maas\_lib.utils package
=======================

.. automodule:: maas_lib.utils
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

maas\_lib.utils.config module
-----------------------------

.. automodule:: maas_lib.utils.config
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.constant module
-------------------------------

.. automodule:: maas_lib.utils.constant
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.logger module
-----------------------------

.. automodule:: maas_lib.utils.logger
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.pymod module
----------------------------

.. automodule:: maas_lib.utils.pymod
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.registry module
-------------------------------

.. automodule:: maas_lib.utils.registry
:members:
:undoc-members:
:show-inheritance:

maas\_lib.utils.type\_assert module
-----------------------------------

.. automodule:: maas_lib.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

+ 34
- 0
docs/source/api/modelscope.fileio.format.rst View File

@@ -0,0 +1,34 @@
modelscope.fileio.format package
================================

.. automodule:: modelscope.fileio.format
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.fileio.format.base module
------------------------------------

.. automodule:: modelscope.fileio.format.base
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.format.json module
------------------------------------

.. automodule:: modelscope.fileio.format.json
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.format.yaml module
------------------------------------

.. automodule:: modelscope.fileio.format.yaml
:members:
:undoc-members:
:show-inheritance:

+ 34
- 0
docs/source/api/modelscope.fileio.rst View File

@@ -0,0 +1,34 @@
modelscope.fileio package
=========================

.. automodule:: modelscope.fileio
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.fileio.format

Submodules
----------

modelscope.fileio.file module
-----------------------------

.. automodule:: modelscope.fileio.file
:members:
:undoc-members:
:show-inheritance:

modelscope.fileio.io module
---------------------------

.. automodule:: modelscope.fileio.io
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.models.cv.cartoon.facelib.LK.rst View File

@@ -0,0 +1,18 @@
modelscope.models.cv.cartoon.facelib.LK package
===============================================

.. automodule:: modelscope.models.cv.cartoon.facelib.LK
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.cv.cartoon.facelib.LK.lk module
-------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.LK.lk
:members:
:undoc-members:
:show-inheritance:

+ 50
- 0
docs/source/api/modelscope.models.cv.cartoon.facelib.rst View File

@@ -0,0 +1,50 @@
modelscope.models.cv.cartoon.facelib package
============================================

.. automodule:: modelscope.models.cv.cartoon.facelib
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.facelib.LK

Submodules
----------

modelscope.models.cv.cartoon.facelib.config module
--------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.config
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.face\_detector module
----------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.face_detector
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.face\_landmark module
----------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.face_landmark
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.facelib.facer module
-------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.facelib.facer
:members:
:undoc-members:
:show-inheritance:

+ 15
- 0
docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.rst View File

@@ -0,0 +1,15 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch package
===================================================

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.mtcnn_pytorch.src

+ 26
- 0
docs/source/api/modelscope.models.cv.cartoon.mtcnn_pytorch.src.rst View File

@@ -0,0 +1,26 @@
modelscope.models.cv.cartoon.mtcnn\_pytorch.src package
=======================================================

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.cv.cartoon.mtcnn\_pytorch.src.align\_trans module
-------------------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans
:members:
:undoc-members:
:show-inheritance:

modelscope.models.cv.cartoon.mtcnn\_pytorch.src.matlab\_cp2tform module
-----------------------------------------------------------------------

.. automodule:: modelscope.models.cv.cartoon.mtcnn_pytorch.src.matlab_cp2tform
:members:
:undoc-members:
:show-inheritance:

+ 27
- 0
docs/source/api/modelscope.models.cv.cartoon.rst View File

@@ -0,0 +1,27 @@
modelscope.models.cv.cartoon package
====================================

.. automodule:: modelscope.models.cv.cartoon
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon.facelib
modelscope.models.cv.cartoon.mtcnn_pytorch

Submodules
----------

modelscope.models.cv.cartoon.utils module
-----------------------------------------

.. automodule:: modelscope.models.cv.cartoon.utils
:members:
:undoc-members:
:show-inheritance:

+ 15
- 0
docs/source/api/modelscope.models.cv.rst View File

@@ -0,0 +1,15 @@
modelscope.models.cv package
============================

.. automodule:: modelscope.models.cv
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv.cartoon

+ 26
- 0
docs/source/api/modelscope.models.nlp.rst View File

@@ -0,0 +1,26 @@
modelscope.models.nlp package
=============================

.. automodule:: modelscope.models.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.models.nlp.sequence\_classification\_model module
------------------------------------------------------------

.. automodule:: modelscope.models.nlp.sequence_classification_model
:members:
:undoc-members:
:show-inheritance:

modelscope.models.nlp.text\_generation\_model module
----------------------------------------------------

.. automodule:: modelscope.models.nlp.text_generation_model
:members:
:undoc-members:
:show-inheritance:

+ 35
- 0
docs/source/api/modelscope.models.rst View File

@@ -0,0 +1,35 @@
modelscope.models package
=========================

.. automodule:: modelscope.models
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.models.cv
modelscope.models.nlp

Submodules
----------

modelscope.models.base module
-----------------------------

.. automodule:: modelscope.models.base
:members:
:undoc-members:
:show-inheritance:

modelscope.models.builder module
--------------------------------

.. automodule:: modelscope.models.builder
:members:
:undoc-members:
:show-inheritance:

+ 7
- 0
docs/source/api/modelscope.pipelines.audio.rst View File

@@ -0,0 +1,7 @@
modelscope.pipelines.audio package
==================================

.. automodule:: modelscope.pipelines.audio
:members:
:undoc-members:
:show-inheritance:

+ 26
- 0
docs/source/api/modelscope.pipelines.cv.rst View File

@@ -0,0 +1,26 @@
modelscope.pipelines.cv package
===============================

.. automodule:: modelscope.pipelines.cv
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.cv.image\_cartoon\_pipeline module
-------------------------------------------------------

.. automodule:: modelscope.pipelines.cv.image_cartoon_pipeline
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.cv.image\_matting\_pipeline module
-------------------------------------------------------

.. automodule:: modelscope.pipelines.cv.image_matting_pipeline
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.pipelines.multi_modal.rst View File

@@ -0,0 +1,18 @@
modelscope.pipelines.multi\_modal package
=========================================

.. automodule:: modelscope.pipelines.multi_modal
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.multi\_modal.image\_captioning module
----------------------------------------------------------

.. automodule:: modelscope.pipelines.multi_modal.image_captioning
:members:
:undoc-members:
:show-inheritance:

+ 26
- 0
docs/source/api/modelscope.pipelines.nlp.rst View File

@@ -0,0 +1,26 @@
modelscope.pipelines.nlp package
================================

.. automodule:: modelscope.pipelines.nlp
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pipelines.nlp.sequence\_classification\_pipeline module
------------------------------------------------------------------

.. automodule:: modelscope.pipelines.nlp.sequence_classification_pipeline
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.nlp.text\_generation\_pipeline module
----------------------------------------------------------

.. automodule:: modelscope.pipelines.nlp.text_generation_pipeline
:members:
:undoc-members:
:show-inheritance:

+ 53
- 0
docs/source/api/modelscope.pipelines.rst View File

@@ -0,0 +1,53 @@
modelscope.pipelines package
============================

.. automodule:: modelscope.pipelines
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.pipelines.audio
modelscope.pipelines.cv
modelscope.pipelines.multi_modal
modelscope.pipelines.nlp

Submodules
----------

modelscope.pipelines.base module
--------------------------------

.. automodule:: modelscope.pipelines.base
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.builder module
-----------------------------------

.. automodule:: modelscope.pipelines.builder
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.default module
-----------------------------------

.. automodule:: modelscope.pipelines.default
:members:
:undoc-members:
:show-inheritance:

modelscope.pipelines.util module
--------------------------------

.. automodule:: modelscope.pipelines.util
:members:
:undoc-members:
:show-inheritance:

+ 50
- 0
docs/source/api/modelscope.preprocessors.rst View File

@@ -0,0 +1,50 @@
modelscope.preprocessors package
================================

.. automodule:: modelscope.preprocessors
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.preprocessors.base module
------------------------------------

.. automodule:: modelscope.preprocessors.base
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.builder module
---------------------------------------

.. automodule:: modelscope.preprocessors.builder
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.common module
--------------------------------------

.. automodule:: modelscope.preprocessors.common
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.image module
-------------------------------------

.. automodule:: modelscope.preprocessors.image
:members:
:undoc-members:
:show-inheritance:

modelscope.preprocessors.nlp module
-----------------------------------

.. automodule:: modelscope.preprocessors.nlp
:members:
:undoc-members:
:show-inheritance:

+ 18
- 0
docs/source/api/modelscope.pydatasets.rst View File

@@ -0,0 +1,18 @@
modelscope.pydatasets package
=============================

.. automodule:: modelscope.pydatasets
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.pydatasets.py\_dataset module
----------------------------------------

.. automodule:: modelscope.pydatasets.py_dataset
:members:
:undoc-members:
:show-inheritance:

+ 32
- 0
docs/source/api/modelscope.rst View File

@@ -0,0 +1,32 @@
modelscope package
==================

.. automodule:: modelscope
:members:
:undoc-members:
:show-inheritance:

Subpackages
-----------

.. toctree::
:maxdepth: 4

modelscope.fileio
modelscope.models
modelscope.pipelines
modelscope.preprocessors
modelscope.pydatasets
modelscope.trainers
modelscope.utils

Submodules
----------

modelscope.version module
-------------------------

.. automodule:: modelscope.version
:members:
:undoc-members:
:show-inheritance:

docs/source/api/maas_lib.pipelines.nlp.rst → docs/source/api/modelscope.trainers.nlp.rst View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines.nlp package
modelscope.trainers.nlp package
===============================

.. automodule:: maas_lib.pipelines.nlp
.. automodule:: modelscope.trainers.nlp
:members:
:undoc-members:
:show-inheritance:
@@ -9,10 +9,10 @@ maas\_lib.pipelines.nlp package
Submodules
----------

maas\_lib.pipelines.nlp.sequence\_classification\_pipeline module
-----------------------------------------------------------------
modelscope.trainers.nlp.sequence\_classification\_trainer module
----------------------------------------------------------------

.. automodule:: maas_lib.pipelines.nlp.sequence_classification_pipeline
.. automodule:: modelscope.trainers.nlp.sequence_classification_trainer
:members:
:undoc-members:
:show-inheritance:

docs/source/api/maas_lib.pipelines.rst → docs/source/api/modelscope.trainers.rst View File

@@ -1,7 +1,7 @@
maas\_lib.pipelines package
modelscope.trainers package
===========================

.. automodule:: maas_lib.pipelines
.. automodule:: modelscope.trainers
:members:
:undoc-members:
:show-inheritance:
@@ -12,25 +12,23 @@ Subpackages
.. toctree::
:maxdepth: 4

maas_lib.pipelines.cv
maas_lib.pipelines.multi_modal
maas_lib.pipelines.nlp
modelscope.trainers.nlp

Submodules
----------

maas\_lib.pipelines.base module
modelscope.trainers.base module
-------------------------------

.. automodule:: maas_lib.pipelines.base
.. automodule:: modelscope.trainers.base
:members:
:undoc-members:
:show-inheritance:

maas\_lib.pipelines.builder module
modelscope.trainers.builder module
----------------------------------

.. automodule:: maas_lib.pipelines.builder
.. automodule:: modelscope.trainers.builder
:members:
:undoc-members:
:show-inheritance:

+ 66
- 0
docs/source/api/modelscope.utils.rst View File

@@ -0,0 +1,66 @@
modelscope.utils package
========================

.. automodule:: modelscope.utils
:members:
:undoc-members:
:show-inheritance:

Submodules
----------

modelscope.utils.config module
------------------------------

.. automodule:: modelscope.utils.config
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.constant module
--------------------------------

.. automodule:: modelscope.utils.constant
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.hub module
---------------------------

.. automodule:: modelscope.utils.hub
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.logger module
------------------------------

.. automodule:: modelscope.utils.logger
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.pymod module
-----------------------------

.. automodule:: modelscope.utils.pymod
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.registry module
--------------------------------

.. automodule:: modelscope.utils.registry
:members:
:undoc-members:
:show-inheritance:

modelscope.utils.type\_assert module
------------------------------------

.. automodule:: modelscope.utils.type_assert
:members:
:undoc-members:
:show-inheritance:

+ 3
- 3
docs/source/api/modules.rst View File

@@ -1,7 +1,7 @@
maas_lib
========
modelscope
==========

.. toctree::
:maxdepth: 4

maas_lib
modelscope

+ 5
- 5
docs/source/conf.py View File

@@ -18,10 +18,10 @@ import sphinx_rtd_theme
sys.path.insert(0, os.path.abspath('../../'))
# -- Project information -----------------------------------------------------

project = 'maas_lib'
copyright = '2022-2023, Alibaba MaaS'
author = 'maas_lib Authors'
version_file = '../../maas_lib/version.py'
project = 'modelscope'
copyright = '2022-2023, Alibaba ModelScope'
author = 'modelscope Authors'
version_file = '../../modelscope/version.py'


def get_version():
@@ -88,7 +88,7 @@ html_static_path = ['_static']

# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'maas_lib_doc'
htmlhelp_basename = 'modelscope_doc'

# -- Extension configuration -------------------------------------------------
# Ignore >>> when copying code


+ 8
- 8
docs/source/index.rst View File

@@ -1,11 +1,11 @@
.. maas_lib documentation file,
.. modelscope documentation file,
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.

MaasLib DOCUMENTATION
ModelScope DOCUMENTATION
=======================================

MaasLib doc
ModelScope doc

.. toctree::
:maxdepth: 2
@@ -30,11 +30,11 @@ MaasLib doc
:maxdepth: 10
:caption: API Doc

api/maas_lib.preprocessors
api/maas_lib.models
api/maas_lib.pipelines
api/maas_lib.fileio
api/maas_lib.utils
api/modelscope.preprocessors
api/modelscope.models
api/modelscope.pipelines
api/modelscope.fileio
api/modelscope.utils


Indices and tables


+ 16
- 16
docs/source/quick_start.md View File

@@ -5,39 +5,39 @@

安装完成后,执行如下命令为maas library创建对应的python环境。
```shell
conda create -n maas python=3.6
conda activate maas
conda create -n modelscope python=3.6
conda activate modelscope
```
检查python和pip命令是否切换到conda环境下。
```shell
which python
# ~/workspace/anaconda3/envs/maas/bin/python
# ~/workspace/anaconda3/envs/modelscope/bin/python

which pip
# ~/workspace/anaconda3/envs/maas/bin/pip
# ~/workspace/anaconda3/envs/modelscope/bin/pip
```
注: 本项目只支持`python3`环境,请勿使用python2环境。

## 第三方依赖安装

MaaS Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架:
ModelScope Library目前支持tensorflow,pytorch两大深度学习框架进行模型训练、推理, 在Python 3.6+, Pytorch 1.8+, Tensorflow 2.6上测试可运行,用户可以根据所选模型对应的计算框架进行安装,可以参考如下链接进行安装所需框架:

* [Pytorch安装指导](https://pytorch.org/get-started/locally/)
* [Tensorflow安装指导](https://www.tensorflow.org/install/pip)


## MaaS library 安装
## ModelScope library 安装

注: 如果在安装过程中遇到错误,请前往[常见问题](faq.md)查找解决方案。

### pip安装
```shell
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/maas.txt
pip install -r http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/release/maas/modelscope.txt
```

安装成功后,可以执行如下命令进行验证安装是否正确
```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
```


@@ -45,11 +45,11 @@ python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting

适合本地开发调试使用,修改源码后可以直接执行
```shell
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git maaslib
git clone git@gitlab.alibaba-inc.com:Ali-MaaS/MaaS-lib.git modelscope
git fetch origin master
git checkout master

cd maaslib
cd modelscope

#安装依赖
pip install -r requirements.txt
@@ -60,7 +60,7 @@ export PYTHONPATH=`pwd`

安装成功后,可以执行如下命令进行验证安装是否正确
```shell
python -c "from maas_lib.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
python -c "from modelscope.pipelines import pipeline;print(pipeline('image-matting',model='damo/image-matting-person')('http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png'))"
```


@@ -79,8 +79,8 @@ pipeline函数提供了简洁的推理接口,示例如下, 更多pipeline介
```python
import cv2
import os.path as osp
from maas_lib.pipelines import pipeline
from maas_lib.utils.constant import Tasks
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks

# 根据任务名创建pipeline
img_matting = pipeline(Tasks.image_matting, model='damo/image-matting-person')
@@ -99,9 +99,9 @@ print(f'Output written to {osp.abspath("result.png")}')
```python
import cv2
import os.path as osp
from maas_lib.pipelines import pipeline
from maas_lib.utils.constant import Tasks
from maas_lib.pydatasets import PyDataset
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks
from modelscope.pydatasets import PyDataset

# 使用图像url构建PyDataset,此处也可通过 input_location = '/dir/to/images' 来使用本地文件夹
input_location = [


+ 4
- 4
docs/source/tutorials/pipeline.md View File

@@ -19,7 +19,7 @@
1. pipeline函数支持指定特定任务名称,加载任务默认模型,创建对应Pipeline对象
执行如下python代码
```python
>>> from maas_lib.pipelines import pipeline
>>> from modelscope.pipelines import pipeline
>>> img_matting = pipeline(task='image-matting', model='damo/image-matting-person')
```

@@ -65,8 +65,8 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz

创建tokenizer和模型
```python
>>> from maas_lib.models import Model
>>> from maas_lib.preprocessors import SequenceClassificationPreprocessor
>>> from modelscope.models import Model
>>> from modelscope.preprocessors import SequenceClassificationPreprocessor
>>> model = Model.from_pretrained('damo/bert-base-sst2')
>>> tokenizer = SequenceClassificationPreprocessor(
model.model_dir, first_sequence='sentence', second_sequence=None)
@@ -74,7 +74,7 @@ wget https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/easynlp_modelz

使用tokenizer和模型对象创建pipeline
```python
>>> from maas_lib.pipelines import pipeline
>>> from modelscope.pipelines import pipeline
>>> semantic_cls = pipeline('text-classification', model=model, preprocessor=tokenizer)
>>> semantic_cls("Hello world!")
```


+ 0
- 48
maas_lib/pipelines/default.py View File

@@ -1,48 +0,0 @@
# Copyright (c) Alibaba, Inc. and its affiliates.

from maas_lib.utils.constant import Tasks

DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
}


def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.

Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'

DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)


def get_default_pipeline_info(task):
""" Get default info for certain task.

Args:
task (str): task name.

Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""
assert task in DEFAULT_MODEL_FOR_PIPELINE, \
f'No default pipeline is registered for Task {task}'

pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

maas_lib/__init__.py → modelscope/__init__.py View File


maas_lib/fileio/__init__.py → modelscope/fileio/__init__.py View File


maas_lib/fileio/file.py → modelscope/fileio/file.py View File


maas_lib/fileio/format/__init__.py → modelscope/fileio/format/__init__.py View File


maas_lib/fileio/format/base.py → modelscope/fileio/format/base.py View File


maas_lib/fileio/format/json.py → modelscope/fileio/format/json.py View File


maas_lib/fileio/format/yaml.py → modelscope/fileio/format/yaml.py View File


maas_lib/fileio/io.py → modelscope/fileio/io.py View File


maas_lib/models/__init__.py → modelscope/models/__init__.py View File


maas_lib/models/base.py → modelscope/models/base.py View File

@@ -7,10 +7,10 @@ from typing import Dict, List, Tuple, Union
from maas_hub.file_download import model_file_download
from maas_hub.snapshot_download import snapshot_download

from maas_lib.models.builder import build_model
from maas_lib.utils.config import Config
from maas_lib.utils.constant import CONFIGFILE
from maas_lib.utils.hub import get_model_cache_dir
from modelscope.models.builder import build_model
from modelscope.utils.config import Config
from modelscope.utils.constant import CONFIGFILE
from modelscope.utils.hub import get_model_cache_dir

Tensor = Union['torch.Tensor', 'tf.Tensor']


maas_lib/models/builder.py → modelscope/models/builder.py View File

@@ -1,7 +1,7 @@
# Copyright (c) Alibaba, Inc. and its affiliates.

from maas_lib.utils.config import ConfigDict
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.utils.config import ConfigDict
from modelscope.utils.registry import Registry, build_from_cfg

MODELS = Registry('models')


maas_lib/models/cv/__init__.py → modelscope/models/cv/__init__.py View File


maas_lib/models/cv/cartoon/__init__.py → modelscope/models/cv/cartoon/__init__.py View File


maas_lib/models/cv/cartoon/facelib/LICENSE → modelscope/models/cv/cartoon/facelib/LICENSE View File


maas_lib/models/cv/cartoon/facelib/LK/__init__.py → modelscope/models/cv/cartoon/facelib/LK/__init__.py View File


maas_lib/models/cv/cartoon/facelib/LK/lk.py → modelscope/models/cv/cartoon/facelib/LK/lk.py View File


maas_lib/models/cv/cartoon/facelib/__init__.py → modelscope/models/cv/cartoon/facelib/__init__.py View File


maas_lib/models/cv/cartoon/facelib/config.py → modelscope/models/cv/cartoon/facelib/config.py View File


maas_lib/models/cv/cartoon/facelib/face_detector.py → modelscope/models/cv/cartoon/facelib/face_detector.py View File


maas_lib/models/cv/cartoon/facelib/face_landmark.py → modelscope/models/cv/cartoon/facelib/face_landmark.py View File


maas_lib/models/cv/cartoon/facelib/facer.py → modelscope/models/cv/cartoon/facelib/facer.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/LICENSE → modelscope/models/cv/cartoon/mtcnn_pytorch/LICENSE View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/README.md → modelscope/models/cv/cartoon/mtcnn_pytorch/README.md View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/__init__.py → modelscope/models/cv/cartoon/mtcnn_pytorch/__init__.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/__init__.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/__init__.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/align_trans.py View File


maas_lib/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py → modelscope/models/cv/cartoon/mtcnn_pytorch/src/matlab_cp2tform.py View File


maas_lib/models/cv/cartoon/utils.py → modelscope/models/cv/cartoon/utils.py View File


maas_lib/models/nlp/__init__.py → modelscope/models/nlp/__init__.py View File


maas_lib/models/nlp/sequence_classification_model.py → modelscope/models/nlp/sequence_classification_model.py View File

@@ -2,7 +2,7 @@ from typing import Any, Dict

import numpy as np

from maas_lib.utils.constant import Tasks
from modelscope.utils.constant import Tasks
from ..base import Model
from ..builder import MODELS


maas_lib/models/nlp/text_generation_model.py → modelscope/models/nlp/text_generation_model.py View File

@@ -1,6 +1,6 @@
from typing import Any, Dict

from maas_lib.utils.constant import Tasks
from modelscope.utils.constant import Tasks
from ..base import Model, Tensor
from ..builder import MODELS


maas_lib/pipelines/__init__.py → modelscope/pipelines/__init__.py View File


maas_lib/pipelines/audio/__init__.py → modelscope/pipelines/audio/__init__.py View File


maas_lib/pipelines/base.py → modelscope/pipelines/base.py View File

@@ -6,11 +6,11 @@ from typing import Any, Dict, Generator, List, Union

from maas_hub.snapshot_download import snapshot_download

from maas_lib.models.base import Model
from maas_lib.preprocessors import Preprocessor
from maas_lib.pydatasets import PyDataset
from maas_lib.utils.config import Config
from maas_lib.utils.hub import get_model_cache_dir
from modelscope.models.base import Model
from modelscope.preprocessors import Preprocessor
from modelscope.pydatasets import PyDataset
from modelscope.utils.config import Config
from modelscope.utils.hub import get_model_cache_dir
from .util import is_model_name

Tensor = Union['torch.Tensor', 'tf.Tensor']
@@ -75,7 +75,7 @@ class Pipeline(ABC):
def __call__(self, input: Union[Input, List[Input]], *args,
**post_kwargs) -> Union[Dict[str, Any], Generator]:
# model provider should leave it as it is
# maas library developer will handle this function
# modelscope library developer will handle this function

# simple showcase, need to support iterator type for both tensorflow and pytorch
# input_dict = self._handle_input(input)

maas_lib/pipelines/builder.py → modelscope/pipelines/builder.py View File

@@ -6,16 +6,26 @@ from typing import Union
import json
from maas_hub.file_download import model_file_download

from maas_lib.models.base import Model
from maas_lib.utils.config import Config, ConfigDict
from maas_lib.utils.constant import CONFIGFILE, Tasks
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.models.base import Model
from modelscope.utils.config import Config, ConfigDict
from modelscope.utils.constant import CONFIGFILE, Tasks
from modelscope.utils.registry import Registry, build_from_cfg
from .base import Pipeline
from .default import DEFAULT_MODEL_FOR_PIPELINE, get_default_pipeline_info
from .util import is_model_name

PIPELINES = Registry('pipelines')

DEFAULT_MODEL_FOR_PIPELINE = {
# TaskName: (pipeline_module_name, model_repo)
Tasks.image_matting: ('image-matting', 'damo/image-matting-person'),
Tasks.text_classification:
('bert-sentiment-analysis', 'damo/bert-base-sst2'),
Tasks.text_generation: ('palm', 'damo/nlp_palm_text-generation_chinese'),
Tasks.image_captioning: ('ofa', None),
Tasks.image_generation:
('cv_unet_person-image-cartoon', 'damo/cv_unet_image-matting_damo'),
}


def build_pipeline(cfg: ConfigDict,
task_name: str = None,
@@ -84,3 +94,42 @@ def pipeline(task: str = None,
cfg.preprocessor = preprocessor

return build_pipeline(cfg, task_name=task)


def add_default_pipeline_info(task: str,
model_name: str,
modelhub_name: str = None,
overwrite: bool = False):
""" Add default model for a task.

Args:
task (str): task name.
model_name (str): model_name.
modelhub_name (str): name for default modelhub.
overwrite (bool): overwrite default info.
"""
if not overwrite:
assert task not in DEFAULT_MODEL_FOR_PIPELINE, \
f'task {task} already has default model.'

DEFAULT_MODEL_FOR_PIPELINE[task] = (model_name, modelhub_name)


def get_default_pipeline_info(task):
""" Get default info for certain task.

Args:
task (str): task name.

Return:
A tuple: first element is pipeline name(model_name), second element
is modelhub name.
"""

if task not in DEFAULT_MODEL_FOR_PIPELINE:
# support pipeline which does not register default model
pipeline_name = list(PIPELINES.modules[task].keys())[0]
default_model = None
else:
pipeline_name, default_model = DEFAULT_MODEL_FOR_PIPELINE[task]
return pipeline_name, default_model

maas_lib/pipelines/cv/__init__.py → modelscope/pipelines/cv/__init__.py View File


maas_lib/pipelines/cv/image_cartoon_pipeline.py → modelscope/pipelines/cv/image_cartoon_pipeline.py View File

@@ -6,14 +6,14 @@ import numpy as np
import PIL
import tensorflow as tf

from maas_lib.models.cv.cartoon.facelib.facer import FaceAna
from maas_lib.models.cv.cartoon.mtcnn_pytorch.src.align_trans import (
from modelscope.models.cv.cartoon.facelib.facer import FaceAna
from modelscope.models.cv.cartoon.mtcnn_pytorch.src.align_trans import (
get_reference_facial_points, warp_and_crop_face)
from maas_lib.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size
from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.models.cv.cartoon.utils import get_f5p, padTo16x, resize_size
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline
from ..builder import PIPELINES


maas_lib/pipelines/cv/image_matting_pipeline.py → modelscope/pipelines/cv/image_matting_pipeline.py View File

@@ -5,10 +5,10 @@ import cv2
import numpy as np
import PIL

from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline
from ..builder import PIPELINES


maas_lib/pipelines/multi_modal/__init__.py → modelscope/pipelines/multi_modal/__init__.py View File


maas_lib/pipelines/multi_modal/image_captioning.py → modelscope/pipelines/multi_modal/image_captioning.py View File

@@ -4,10 +4,10 @@ import numpy as np
import torch
from PIL import Image

from maas_lib.pipelines.base import Input
from maas_lib.preprocessors import load_image
from maas_lib.utils.constant import Tasks
from maas_lib.utils.logger import get_logger
from modelscope.pipelines.base import Input
from modelscope.preprocessors import load_image
from modelscope.utils.constant import Tasks
from modelscope.utils.logger import get_logger
from ..base import Pipeline
from ..builder import PIPELINES


maas_lib/pipelines/nlp/__init__.py → modelscope/pipelines/nlp/__init__.py View File


maas_lib/pipelines/nlp/sequence_classification_pipeline.py → modelscope/pipelines/nlp/sequence_classification_pipeline.py View File

@@ -5,9 +5,9 @@ from typing import Any, Dict, Union
import json
import numpy as np

from maas_lib.models.nlp import BertForSequenceClassification
from maas_lib.preprocessors import SequenceClassificationPreprocessor
from maas_lib.utils.constant import Tasks
from modelscope.models.nlp import BertForSequenceClassification
from modelscope.preprocessors import SequenceClassificationPreprocessor
from modelscope.utils.constant import Tasks
from ...models import Model
from ..base import Input, Pipeline
from ..builder import PIPELINES

maas_lib/pipelines/nlp/text_generation_pipeline.py → modelscope/pipelines/nlp/text_generation_pipeline.py View File

@@ -1,9 +1,9 @@
from typing import Dict, Optional, Union

from maas_lib.models import Model
from maas_lib.models.nlp import PalmForTextGenerationModel
from maas_lib.preprocessors import TextGenerationPreprocessor
from maas_lib.utils.constant import Tasks
from modelscope.models import Model
from modelscope.models.nlp import PalmForTextGenerationModel
from modelscope.preprocessors import TextGenerationPreprocessor
from modelscope.utils.constant import Tasks
from ..base import Pipeline, Tensor
from ..builder import PIPELINES


maas_lib/pipelines/util.py → modelscope/pipelines/util.py View File

@@ -5,7 +5,7 @@ import os.path as osp
import json
from maas_hub.file_download import model_file_download

from maas_lib.utils.constant import CONFIGFILE
from modelscope.utils.constant import CONFIGFILE


def is_model_name(model):

maas_lib/preprocessors/__init__.py → modelscope/preprocessors/__init__.py View File


maas_lib/preprocessors/base.py → modelscope/preprocessors/base.py View File


maas_lib/preprocessors/builder.py → modelscope/preprocessors/builder.py View File

@@ -1,8 +1,8 @@
# Copyright (c) Alibaba, Inc. and its affiliates.

from maas_lib.utils.config import ConfigDict
from maas_lib.utils.constant import Fields
from maas_lib.utils.registry import Registry, build_from_cfg
from modelscope.utils.config import ConfigDict
from modelscope.utils.constant import Fields
from modelscope.utils.registry import Registry, build_from_cfg

PREPROCESSORS = Registry('preprocessors')


maas_lib/preprocessors/common.py → modelscope/preprocessors/common.py View File


maas_lib/preprocessors/image.py → modelscope/preprocessors/image.py View File

@@ -4,8 +4,8 @@ from typing import Dict, Union

from PIL import Image, ImageOps

from maas_lib.fileio import File
from maas_lib.utils.constant import Fields
from modelscope.fileio import File
from modelscope.utils.constant import Fields
from .builder import PREPROCESSORS



maas_lib/preprocessors/nlp.py → modelscope/preprocessors/nlp.py View File

@@ -5,8 +5,8 @@ from typing import Any, Dict, Union

from transformers import AutoTokenizer

from maas_lib.utils.constant import Fields, InputFields
from maas_lib.utils.type_assert import type_assert
from modelscope.utils.constant import Fields, InputFields
from modelscope.utils.type_assert import type_assert
from .base import Preprocessor
from .builder import PREPROCESSORS


maas_lib/pydatasets/__init__.py → modelscope/pydatasets/__init__.py View File


maas_lib/pydatasets/py_dataset.py → modelscope/pydatasets/py_dataset.py View File

@@ -4,7 +4,7 @@ from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,

from datasets import Dataset, load_dataset

from maas_lib.utils.logger import get_logger
from modelscope.utils.logger import get_logger

logger = get_logger()

@@ -52,7 +52,7 @@ class PyDataset:
Mapping[str, Union[str,
Sequence[str]]]]] = None
) -> 'PyDataset':
"""Load a PyDataset from the MaaS Hub, Hugging Face Hub, urls, or a local dataset.
"""Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset.
Args:

path (str): Path or name of the dataset.

maas_lib/tools/eval.py → modelscope/tools/eval.py View File

@@ -2,7 +2,7 @@

import argparse

from maas_lib.trainers import build_trainer
from modelscope.trainers import build_trainer


def parse_args():

maas_lib/tools/train.py → modelscope/tools/train.py View File

@@ -2,7 +2,7 @@

import argparse

from maas_lib.trainers import build_trainer
from modelscope.trainers import build_trainer


def parse_args():

maas_lib/trainers/__init__.py → modelscope/trainers/__init__.py View File


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save