diff --git a/README.md b/README.md index 2acb5b64..d7733c67 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,12 @@ Sedna consists of the following components： ## Guides + +### Documents + +Documentation is located on [readthedoc.io](https://sedna.readthedocs.io/). These documents can help you understand Sedna better. + + ### Installation Follow the [Sedna installation document](docs/setup/install.md) to install Sedna. diff --git a/README_zh.md b/README_zh.md index 0072709c..5e093e40 100644 --- a/README_zh.md +++ b/README_zh.md @@ -58,7 +58,10 @@ Sedna由以下组件构建： -## 文档 +## 指南 + +### 文档 +Sedna 在 [readthedoc.io](https://sedna.readthedocs.io/) 托管相关文档。 您可以根据这些文档更好地了解Sedna。 ### 安装 Sedna的安装文档请参考[这里](/docs/setup/install.md)。 diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 385bacaa..9ec252a2 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,3 +1,7 @@ .wy-nav-content{ max-width: 100%; -} \ No newline at end of file +} + +.rst-content .section img { + width: 1024px +} diff --git a/docs/_static/logo.png b/docs/_static/logo.png new file mode 100644 index 00000000..ceb6710f Binary files /dev/null and b/docs/_static/logo.png differ diff --git a/docs/api/crd/index.rst b/docs/api/crd/index.rst new file mode 100644 index 00000000..f5aa4ecb --- /dev/null +++ b/docs/api/crd/index.rst @@ -0,0 +1,3 @@ +=========================================== +Sedna CRD API +=========================================== \ No newline at end of file diff --git a/docs/api/lib/index.rst b/docs/api/lib/index.rst new file mode 100644 index 00000000..15569e51 --- /dev/null +++ b/docs/api/lib/index.rst @@ -0,0 +1,5 @@ +=========================================== +Sedna Python SDK +=========================================== + +.. mdinclude:: ../../../lib/sedna/README.md \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index d739b8c6..a932286f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,11 +11,49 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os +import re import sys +import shutil +import subprocess + import sphinx_rtd_theme -sys.path.insert(0, os.path.abspath('../lib')) -sys.path.insert(0, os.path.abspath('../lib/sedna')) +try: + import m2r2 +except ModuleNotFoundError: + subprocess.check_call([sys.executable, "-m", "pip", "install", "m2r2"]) + +try: + import autoapi +except ModuleNotFoundError: + subprocess.check_call([sys.executable, "-m", "pip", + "install", "sphinx-autoapi"]) + +_base_path = os.path.abspath('..') +BASE_URL = 'https://github.com/kubeedge/sedna/' + +sys.path.append(os.path.join(_base_path, "lib")) +sys.path.append(_base_path) + +extra_paths = [ + os.path.join(_base_path, "examples"), +] +for p in extra_paths: + dst = os.path.join( + _base_path, "docs", + os.path.basename(p) + ) + if os.path.isfile(dst): + os.remove(dst) + elif os.path.isdir(dst): + shutil.rmtree(dst) + if os.path.isdir(p): + shutil.copytree(p, dst) + else: + shutil.copy2(p, dst) + +with open(f'{_base_path}/lib/sedna/VERSION', "r", encoding="utf-8") as fh: + __version__ = fh.read().strip() # -- Project information ----------------------------------------------------- @@ -23,17 +61,30 @@ project = 'Sedna' copyright = '2020, Kubeedge' author = 'Kubeedge' +version = __version__ +release = __version__ # -- General configuration --------------------------------------------------- -from recommonmark.parser import CommonMarkParser -source_parsers = { - '.md': CommonMarkParser, -} # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['m2r2', 'sphinx.ext.autodoc', 'sphinx_markdown_tables', ] +extensions = [ + "m2r2", + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "autoapi.extension", + "sphinx.ext.intersphinx", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon" +] + +autodoc_inherit_docstrings = False +autodoc_member_order = "bysource" +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True # Add any paths that contain templates here, relative to this directory. # templates_path = ['_templates'] @@ -58,7 +109,7 @@ html_static_path = ['_static'] # html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - +html_last_updated_fmt = "%b %d, %Y" html_theme_options = { 'prev_next_buttons_location': 'both' } @@ -74,6 +125,101 @@ source_suffix = { '.md': 'markdown', } +autoapi_type = "python" +autoapi_dirs = [f"{_base_path}/lib/sedna"] +autoapi_options = [ + 'members', 'undoc-members', 'show-inheritance', + 'show-module-summary', 'special-members', 'imported-members' +] + +extlinks = { + "issue": f"{BASE_URL}issues", + "pr": f"{BASE_URL}pull" +} + + +# hack to replace file link to html link in markdown +def ultimateReplace(app, docname, source): + """ + In the rendering with Sphinx, as some file links in markdown + can not be automatically redirected, and 404 response during + access, here define a regular to handle these links. + """ + path = app.env.doc2path(docname) # get current path + + INLINE_LINK_RE = re.compile(r'\[[^\]]+\]\(([^)]+)\)') + FOOTNOTE_LINK_URL_RE = re.compile(r'\[[^\]]+\](?:\s+)?:(?:\s+)?(\S+)') + if path.endswith('.md'): + new_line = [] + + docs_url = os.path.join(_base_path, "docs") + for line in source[0].split('\n'): + line = re.sub( + "\[`([^\]]+)`\]\[", "[\g<1>][", line + ) # fix html render error: [`title`] + replace_line = [] + prev_start = 0 + href_list = ( + list(INLINE_LINK_RE.finditer(line)) + + list(FOOTNOTE_LINK_URL_RE.finditer(line)) + ) + for href in href_list: + pstart = href.start(1) + pstop = href.end(1) + if pstart == -1 or pstop == -1: + continue + link = line[pstart: pstop] + if not link or link.startswith("http"): + continue + if link.startswith("/"): + tmp = _base_path + else: + tmp = os.path.abspath(os.path.dirname(path)) + + _relpath = os.path.abspath(os.path.join(tmp, link.lstrip("/"))) + for sp in extra_paths: # these docs will move into `docs` + sp = os.path.abspath(sp).rstrip("/") + + if not _relpath.startswith(sp): + continue + if os.path.isdir(sp): + sp += "/" + _relpath = os.path.join( + docs_url, _relpath[len(_base_path):].lstrip("/") + ) + break + + if _relpath.startswith(docs_url) and ( + os.path.isdir(_relpath) or + os.path.splitext(_relpath)[-1].lower().startswith( + ( + ".md", ".rst", ".txt", "html", + ".png", ".jpg", ".jpeg", ".svg", ".gif" + ) + ) + ): + link = os.path.relpath(_relpath, + os.path.dirname(path)) + if not os.path.isdir(_relpath): # suffix edit + link = re.sub( + "(?:\.md|\.rst|\.txt)(\W+\w+)?$", + ".html\g<1>", link + ) + else: # redirect to `github` + _relpath = os.path.abspath( + os.path.join(tmp, link.lstrip("/")) + ) + _rel_root = os.path.relpath(_relpath, _base_path) + link = f"{BASE_URL}tree/main/{_rel_root}" + p_line = f"{line[prev_start:pstart]}{link}" + prev_start = pstop + replace_line.append(p_line) + replace_line.append(line[prev_start:]) + new_line.append("".join(replace_line)) + source[0] = "\n".join(new_line) + def setup(app): - app.add_stylesheet('css/custom.css') + app.add_config_value('ultimate_replacements', {}, True) + app.connect('source-read', ultimateReplace) + app.add_css_file('css/custom.css') diff --git a/docs/examples/joint_inference_example_link.rst b/docs/examples/joint_inference_example_link.rst deleted file mode 100644 index 4f17102d..00000000 --- a/docs/examples/joint_inference_example_link.rst +++ /dev/null @@ -1 +0,0 @@ -.. mdinclude:: ../../examples/helmet_detection_inference/README.md diff --git a/docs/index.rst b/docs/index.rst index 33678893..99d5b278 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,12 +2,16 @@ Sedna documentation =========================================== +.. image:: ./_static/logo.png + :width: 200 +Sedna is an edge-cloud synergy AI project incubated in KubeEdge SIG AI. Benefiting from the edge-cloud synergy capabilities provided by KubeEdge, Sedna can implement across edge-cloud collaborative training and collaborative inference capabilities, such as joint inference, incremental learning, federated learning, and lifelong learning. Sedna supports popular AI frameworks, such as TensorFlow, Pytorch, PaddlePaddle, MindSpore. + +Sedna can simply enable edge-cloud synergy capabilities to existing training and inference scripts, bringing the benefits of reducing costs, improving model performance, and protecting data privacy. .. toctree:: :maxdepth: 1 :caption: QUICK START - :hidden: quickstart @@ -19,35 +23,70 @@ Sedna documentation proposals/architecture proposals/dataset-and-model + proposals/federated-learning + proposals/incremental-learning proposals/joint-inference + proposals/lifelong-learning + proposals/object-search + proposals/object-tracking .. toctree:: - :maxdepth: 1 - :caption: Setup - - setup/install + :maxdepth: 1 + :titlesonly: + :glob: + :caption: DEPLOY + Installtion + Standalone .. toctree:: :maxdepth: 1 + :glob: :caption: EXAMPLES - :hidden: - examples/joint_inference_example_link + examples/federated_learning/surface_defect_detection/README + examples/incremental_learning/helmet_detection/README + examples/joint_inference/helmet_detection_inference/README + examples/lifelong_learning/atcii/README + examples/storage/s3/* .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: API - :hidden: + :titlesonly: + :glob: + + api/crd/* + api/lib/* + +.. toctree:: + :caption: Contributing + + Prepare + - lib-api/modules +.. toctree:: + :maxdepth: 1 + :caption: API REFERENCE + :titlesonly: + :glob: + autoapi/lib/sedna/index + + +.. toctree:: + :caption: ROADMAP + :hidden: + roadmap +RELATED LINKS +============= +.. mdinclude:: related_link.md Indices and tables ================== diff --git a/docs/lib-api/modules.rst b/docs/lib-api/modules.rst deleted file mode 100644 index b846b99e..00000000 --- a/docs/lib-api/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -sedna -======= - -.. toctree:: - :maxdepth: 4 - - sedna diff --git a/docs/lib-api/sedna.hard_example_mining.image_classification.rst b/docs/lib-api/sedna.hard_example_mining.image_classification.rst deleted file mode 100644 index 2f03c5fb..00000000 --- a/docs/lib-api/sedna.hard_example_mining.image_classification.rst +++ /dev/null @@ -1,16 +0,0 @@ -sedna.hard\_example\_mining.image\_classification package -=========================================================== - -.. automodule:: sedna.hard_example_mining.image_classification - :members: - :undoc-members: - :show-inheritance: - - -sedna.hard\_example\_mining.image\_classification.hard\_mine\_filters module ------------------------------------------------------------------------------- - -.. automodule:: sedna.hard_example_mining.image_classification.hard_mine_filters - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/lib-api/sedna.hard_example_mining.object_detection.rst b/docs/lib-api/sedna.hard_example_mining.object_detection.rst deleted file mode 100644 index 2d0cbbc7..00000000 --- a/docs/lib-api/sedna.hard_example_mining.object_detection.rst +++ /dev/null @@ -1,18 +0,0 @@ -sedna.hard\_example\_mining.object\_detection package -======================================================= - -.. automodule:: sedna.hard_example_mining.object_detection - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -sedna.hard\_example\_mining.object\_detection.scores\_filters module ----------------------------------------------------------------------- - -.. automodule:: sedna.hard_example_mining.object_detection.scores_filters - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/lib-api/sedna.hard_example_mining.rst b/docs/lib-api/sedna.hard_example_mining.rst deleted file mode 100644 index b5a52e00..00000000 --- a/docs/lib-api/sedna.hard_example_mining.rst +++ /dev/null @@ -1,27 +0,0 @@ -sedna.hard\_example\_mining package -===================================== - -.. automodule:: sedna.hard_example_mining - :members: - :undoc-members: - :show-inheritance: - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - sedna.hard_example_mining.image_classification - sedna.hard_example_mining.object_detection - -Submodules ----------- - -sedna.hard\_example\_mining.base module ------------------------------------------ - -.. automodule:: sedna.hard_example_mining.base - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/lib-api/sedna.joint_inference.rst b/docs/lib-api/sedna.joint_inference.rst deleted file mode 100644 index b0cfc992..00000000 --- a/docs/lib-api/sedna.joint_inference.rst +++ /dev/null @@ -1,26 +0,0 @@ -sedna.joint\_inference package -================================ - -.. automodule:: sedna.joint_inference - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -sedna.joint\_inference.data module ------------------------------------- - -.. automodule:: sedna.joint_inference.data - :members: - :undoc-members: - :show-inheritance: - -sedna.joint\_inference.joint\_inference module ------------------------------------------------- - -.. automodule:: sedna.joint_inference.joint_inference - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/lib-api/sedna.rst b/docs/lib-api/sedna.rst deleted file mode 100644 index bf0d9a5b..00000000 --- a/docs/lib-api/sedna.rst +++ /dev/null @@ -1,18 +0,0 @@ - -Subpackages ------------ - -.. toctree:: - :maxdepth: 4 - - sedna.hard_example_mining - sedna.joint_inference - -sedna.context module ----------------------- - -.. automodule:: sedna.context - :members: - :undoc-members: - :show-inheritance: - diff --git a/docs/proposals/dataset-and-model.md b/docs/proposals/dataset-and-model.md index a05bd510..08dec759 100644 --- a/docs/proposals/dataset-and-model.md +++ b/docs/proposals/dataset-and-model.md @@ -73,7 +73,7 @@ The tables below summarize the group, kind and API version details for the CRDs. #### `Dataset` CRD -[crd source](/build/crds/sedna/dataset_v1alpha1.yaml) +[crd source](/build/crds/sedna.io_datasets.yaml) ```yaml apiVersion: apiextensions.k8s.io/v1 @@ -144,7 +144,7 @@ Current we support these below formats: #### `Model` CRD -[crd source](/build/crds/sedna/model_v1alpha1.yaml) +[crd source](/build/crds/sedna.io_models.yaml) ```yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -209,7 +209,7 @@ spec: ### CRD type definition - `Dataset` -[go source](cloud/pkg/apis/sedna/v1alpha1/dataset_types.go) +[go source](/pkg/apis/sedna/v1alpha1/dataset_types.go) ```go package v1alpha1 @@ -259,7 +259,7 @@ type DatasetList struct { - `Model` -[go source](cloud/pkg/apis/sedna/v1alpha1/model_types.go) +[go source](/pkg/apis/sedna/v1alpha1/model_types.go) ```go package v1alpha1 diff --git a/docs/proposals/incremental-learning.md b/docs/proposals/incremental-learning.md index a523794b..59ebbd3c 100644 --- a/docs/proposals/incremental-learning.md +++ b/docs/proposals/incremental-learning.md @@ -74,7 +74,7 @@ The tables below summarize the group, kind and API version details for the CRD. |Kind | IncrementalLearningJob | ### Incremental learning CRD -See the [crd source](/build/crds/sedna/incrementallearningjob_v1alpha1.yaml) for details. +See the [crd source](/build/crds/sedna.io_incrementallearningjobs.yaml) for details. ### Incremental learning job type definition diff --git a/docs/proposals/joint-inference.md b/docs/proposals/joint-inference.md index c13c64ba..493826ce 100644 --- a/docs/proposals/joint-inference.md +++ b/docs/proposals/joint-inference.md @@ -64,7 +64,7 @@ The tables below summarize the group, kind and API version details for the CRD. ### Joint inference CRD -see [crd source](/build/crd-samples/sedna/jointinferenceservice_v1alpha1.yaml) +see [crd source](/build/crds/sedna.io_jointinferenceservices.yaml) ### Joint inference type definition diff --git a/docs/proposals/object-search.md b/docs/proposals/object-search.md index f5de0bb9..b01428d3 100644 --- a/docs/proposals/object-search.md +++ b/docs/proposals/object-search.md @@ -15,6 +15,7 @@ - [Details of api between GM(cloud) and LC(edge)](#details-of-api-between-gmcloud-and-lcedge) - [Flow of object search service creation](#flow-of-object-search-service-creation) - [Workers Communication](#workers-communication) + # Object Search Service ## Motivation Object search is an important technology in the field of computer vision, which is widely used in security monitoring, intelligent transportation, etc. Generally, online object search applications have stringent latency constraints, which cannot be met by cloud computing schemes. Object search schemes based on edge computing have the characteristics of low latency and data privacy security, and are the mainstream technology trend. diff --git a/docs/proposals/object-tracking.md b/docs/proposals/object-tracking.md index 3bf00d44..12e79fe9 100644 --- a/docs/proposals/object-tracking.md +++ b/docs/proposals/object-tracking.md @@ -15,6 +15,7 @@ - [Details of api between GM(cloud) and LC(edge)](#details-of-api-between-gmcloud-and-lcedge) - [Flow of object tracking service creation](#flow-of-object-tracking-service-creation) - [Workers Communication](#workers-communication) + # Object Tracking Service ## Motivation Object tracking is an important technology in the field of computer vision, which is widely used in security monitoring, intelligent transportation, etc. Generally, object tracking applications have high latency requirements, which cannot be met by cloud computing schemes. The object tracking schemes based on edge computing have the characteristics of low latency and data privacy security, and are the mainstream technology trend. diff --git a/docs/quickstart.md b/docs/quickstart.md index d6c1c59c..7a60b0d1 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,23 +1,50 @@ -## Getting start -Sedna is an open source framework for edge-cloud collaborative training and inference, so that AI applications running at the edge can benefit from cost reduction, model performance improvement and data privacy protection. +# Quick Start + +## Introduction + +Sedna provide some examples of running Sedna jobs in [here](/examples/README.md) + +Here is a general guide to quick start an incremental learning job. ### Get Sedna -You can find the latest Sedna release [here](TODO) +You can find the latest Sedna release [here](https://github.com/kubeedge/sedna/releases). ### Deploying Sedna -Please refer to this [link](setup/install.html). +Please refer to this [link](setup/install.md). + +### Prepare + +todo + +### Configure + +todo + +### Run + +todo + +### Monitor + +todo + +## API + +- CRD: Please refer to this [link](api/crd). +- Lib: Please refer to this [link](api/lib). -### Examples -Please refer to this[link](TODO) +## Contributing -### Contributing +Contributions are very welcome! You can see our [CONTRIBUTING](CONTRIBUTING.md) for more information. -Contributions are very welcome! You can see our [CONTRIBUTING.md](TODO) for more information +## Community -### Community +Sedna is an open source project and in the spirit of openness and freedom, we welcome new contributors to join us. +You can get in touch with the community according to the ways: +* [Github Issues](https://github.com/kubeedge/sedna/issues) +* [Regular Community Meeting](https://zoom.us/j/4167237304) +* [slack channel](https://app.slack.com/client/TDZ5TGXQW/C01EG84REVB/details) -Sedna is an open source project and In the spirit of openness and freedom, we welcome new contributors to join us . You can get in touch with the community according to the ways: -* [Github Issues](TODO) diff --git a/docs/related_link.md b/docs/related_link.md new file mode 100644 index 00000000..e80bc031 --- /dev/null +++ b/docs/related_link.md @@ -0,0 +1 @@ +[支持边云协同终身学习特性，KubeEdge 子项目 Sedna 0.3.0 版本发布！](https://juejin.cn/post/6970866022286884878) \ No newline at end of file diff --git a/examples/README.md b/examples/README.md index 355cc12f..296b58cd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,6 +6,7 @@ This repository is home to the following features of examples: * [Joint Inference](#joint-inference) * [Incremental Learning](#incremental-learning) * [Federated Learning](#federated-learning) +* [Lifelong Learning](#lifelong-learning) * [Shared Storage](#shared-storage) ### Joint Inference @@ -17,6 +18,9 @@ Example: [Using Incremental Learning Job in Helmet Detection Scenario](./increme ### Federated Learning Example: [Using Federated Learning Job in Surface Defect Detection Scenario](./federated_learning/surface_defect_detection/README.md) +### Lifelong Learning +Example: [Using Lifelong Learning Job in Thermal Comfort Prediction Scenario](./lifelong_learning/atcii/README.md) + ### Shared Storage | Support Protocols |Support Features| Examples |Release| | :-------------: | :-------------: |:-------------: | :-------------: |