Merge pull request #150 from JoeyHwong-gk/docs

docs improvement

README.md

@@ -71,6 +71,12 @@ Sedna consists of the following components：
 
 
 ## Guides
+
+### Documents
+
+Documentation is located on [readthedoc.io](https://sedna.readthedocs.io/). These documents can help you understand Sedna better.
+
+
 ### Installation
 Follow the [Sedna installation document](docs/setup/install.md) to install Sedna.
 

README_zh.md

@@ -58,7 +58,10 @@ Sedna由以下组件构建：
 
 
 
-## 文档
+## 指南
+
+### 文档
+Sedna 在 [readthedoc.io](https://sedna.readthedocs.io/) 托管相关文档。 您可以根据这些文档更好地了解Sedna。
 ### 安装
 Sedna的安装文档请参考[这里](/docs/setup/install.md)。
 

docs/_static/css/custom.css

@@ -1,3 +1,7 @@
 .wy-nav-content{
     max-width: 100%;
-}
+}
+
+.rst-content .section img {
+	width: 1024px
+}

docs/api/crd/index.rst

@@ -0,0 +1,3 @@
+===========================================
+Sedna CRD API
+===========================================

docs/api/lib/index.rst

@@ -0,0 +1,5 @@
+===========================================
+Sedna Python SDK
+===========================================
+
+.. mdinclude:: ../../../lib/sedna/README.md

docs/conf.py

@@ -11,11 +11,49 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
+import re
 import sys
+import shutil
+import subprocess
+
 import sphinx_rtd_theme
 
-sys.path.insert(0, os.path.abspath('../lib'))
-sys.path.insert(0, os.path.abspath('../lib/sedna'))
+try:
+    import m2r2
+except ModuleNotFoundError:
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "m2r2"])
+
+try:
+    import autoapi
+except ModuleNotFoundError:
+    subprocess.check_call([sys.executable, "-m", "pip",
+                           "install", "sphinx-autoapi"])
+
+_base_path = os.path.abspath('..')
+BASE_URL = 'https://github.com/kubeedge/sedna/'
+
+sys.path.append(os.path.join(_base_path, "lib"))
+sys.path.append(_base_path)
+
+extra_paths = [
+    os.path.join(_base_path, "examples"),
+]
+for p in extra_paths:
+    dst = os.path.join(
+        _base_path, "docs",
+        os.path.basename(p)
+    )
+    if os.path.isfile(dst):
+        os.remove(dst)
+    elif os.path.isdir(dst):
+        shutil.rmtree(dst)
+    if os.path.isdir(p):
+        shutil.copytree(p, dst)
+    else:
+        shutil.copy2(p, dst)
+
+with open(f'{_base_path}/lib/sedna/VERSION', "r", encoding="utf-8") as fh:
+    __version__ = fh.read().strip()
 
 # -- Project information -----------------------------------------------------
 
@@ -23,17 +61,30 @@ project = 'Sedna'
 copyright = '2020, Kubeedge'
 author = 'Kubeedge'
 
+version = __version__
+release = __version__
 # -- General configuration ---------------------------------------------------
 
-from recommonmark.parser import CommonMarkParser
 
-source_parsers = {
-    '.md': CommonMarkParser,
-}
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['m2r2', 'sphinx.ext.autodoc', 'sphinx_markdown_tables', ]
+extensions = [
+    "m2r2",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.viewcode",
+    "autoapi.extension",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon"
+]
+
+autodoc_inherit_docstrings = False
+autodoc_member_order = "bysource"
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
 
 # Add any paths that contain templates here, relative to this directory.
 # templates_path = ['_templates']
@@ -58,7 +109,7 @@ html_static_path = ['_static']
 #
 html_theme = 'sphinx_rtd_theme'
 html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
+html_last_updated_fmt = "%b %d, %Y"
 html_theme_options = {
     'prev_next_buttons_location': 'both'
 }
@@ -74,6 +125,101 @@ source_suffix = {
     '.md': 'markdown',
 }
 
+autoapi_type = "python"
+autoapi_dirs = [f"{_base_path}/lib/sedna"]
+autoapi_options = [
+    'members', 'undoc-members', 'show-inheritance',
+    'show-module-summary', 'special-members', 'imported-members'
+]
+
+extlinks = {
+    "issue": f"{BASE_URL}issues",
+    "pr": f"{BASE_URL}pull"
+}
+
+
+# hack to replace file link to html link in markdown
+def ultimateReplace(app, docname, source):
+    """
+    In the rendering with Sphinx, as some file links in markdown
+    can not be automatically redirected, and 404 response during
+    access, here define a regular to handle these links.
+    """
+    path = app.env.doc2path(docname)  # get current path
+
+    INLINE_LINK_RE = re.compile(r'\[[^\]]+\]\(([^)]+)\)')
+    FOOTNOTE_LINK_URL_RE = re.compile(r'\[[^\]]+\](?:\s+)?:(?:\s+)?(\S+)')
+    if path.endswith('.md'):
+        new_line = []
+
+        docs_url = os.path.join(_base_path, "docs")
+        for line in source[0].split('\n'):
+            line = re.sub(
+                "\[`([^\]]+)`\]\[", "[\g<1>][", line
+            )  # fix html render error: [`title`]
+            replace_line = []
+            prev_start = 0
+            href_list = (
+                    list(INLINE_LINK_RE.finditer(line)) +
+                    list(FOOTNOTE_LINK_URL_RE.finditer(line))
+            )
+            for href in href_list:
+                pstart = href.start(1)
+                pstop = href.end(1)
+                if pstart == -1 or pstop == -1:
+                    continue
+                link = line[pstart: pstop]
+                if not link or link.startswith("http"):
+                    continue
+                if link.startswith("/"):
+                    tmp = _base_path
+                else:
+                    tmp = os.path.abspath(os.path.dirname(path))
+
+                _relpath = os.path.abspath(os.path.join(tmp, link.lstrip("/")))
+                for sp in extra_paths:  # these docs will move into `docs`
+                    sp = os.path.abspath(sp).rstrip("/")
+
+                    if not _relpath.startswith(sp):
+                        continue
+                    if os.path.isdir(sp):
+                        sp += "/"
+                    _relpath = os.path.join(
+                        docs_url, _relpath[len(_base_path):].lstrip("/")
+                    )
+                    break
+
+                if _relpath.startswith(docs_url) and (
+                        os.path.isdir(_relpath) or
+                        os.path.splitext(_relpath)[-1].lower().startswith(
+                            (
+                                    ".md", ".rst", ".txt", "html",
+                                    ".png", ".jpg", ".jpeg", ".svg", ".gif"
+                            )
+                        )
+                ):
+                    link = os.path.relpath(_relpath,
+                                           os.path.dirname(path))
+                    if not os.path.isdir(_relpath):  # suffix edit
+                        link = re.sub(
+                            "(?:\.md|\.rst|\.txt)(\W+\w+)?$",
+                            ".html\g<1>", link
+                        )
+                else:  # redirect to `github`
+                    _relpath = os.path.abspath(
+                        os.path.join(tmp, link.lstrip("/"))
+                    )
+                    _rel_root = os.path.relpath(_relpath, _base_path)
+                    link = f"{BASE_URL}tree/main/{_rel_root}"
+                p_line = f"{line[prev_start:pstart]}{link}"
+                prev_start = pstop
+                replace_line.append(p_line)
+            replace_line.append(line[prev_start:])
+            new_line.append("".join(replace_line))
+        source[0] = "\n".join(new_line)
+
 
 def setup(app):
-    app.add_stylesheet('css/custom.css')
+    app.add_config_value('ultimate_replacements', {}, True)
+    app.connect('source-read', ultimateReplace)
+    app.add_css_file('css/custom.css')

docs/examples/joint_inference_example_link.rst

@@ -1 +0,0 @@
-.. mdinclude:: ../../examples/helmet_detection_inference/README.md

docs/index.rst

@@ -2,12 +2,16 @@
 Sedna documentation
 ===========================================
 
+.. image:: ./_static/logo.png
+   :width: 200
 
+Sedna is an edge-cloud synergy AI project incubated in KubeEdge SIG AI. Benefiting from the edge-cloud synergy capabilities provided by KubeEdge, Sedna can implement across edge-cloud collaborative training and collaborative inference capabilities, such as joint inference, incremental learning, federated learning, and lifelong learning. Sedna supports popular AI frameworks, such as TensorFlow, Pytorch, PaddlePaddle, MindSpore.
+
+Sedna can simply enable edge-cloud synergy capabilities to existing training and inference scripts, bringing the benefits of reducing costs, improving model performance, and protecting data privacy.
 
 .. toctree::
     :maxdepth: 1
     :caption: QUICK START
-    :hidden:
 
     quickstart
 
@@ -19,35 +23,70 @@ Sedna documentation
 
     proposals/architecture
     proposals/dataset-and-model
+    proposals/federated-learning
+    proposals/incremental-learning
     proposals/joint-inference
+    proposals/lifelong-learning
+    proposals/object-search
+    proposals/object-tracking
 
 
 .. toctree::
-   :maxdepth: 1
-   :caption: Setup
-
-   setup/install
+    :maxdepth: 1
+    :titlesonly:
+    :glob:
+    :caption: DEPLOY
 
+    Installtion <setup/install>
+    Standalone <setup/local-up>
 
 .. toctree::
     :maxdepth: 1
+    :glob:
     :caption: EXAMPLES
-    :hidden:
 
-    examples/joint_inference_example_link
+    examples/federated_learning/surface_defect_detection/README
+    examples/incremental_learning/helmet_detection/README
+    examples/joint_inference/helmet_detection_inference/README
+    examples/lifelong_learning/atcii/README
+    examples/storage/s3/*
 
 
 .. toctree::
-    :maxdepth: 2
+    :maxdepth: 1
     :caption: API
-    :hidden:
+    :titlesonly:
+    :glob:
+
+    api/crd/*
+    api/lib/*
+
+.. toctree::
+    :caption: Contributing
+
+    Prepare <contributing/prepare-environment>
+
 
-    lib-api/modules
+.. toctree::
+    :maxdepth: 1
+    :caption: API REFERENCE
+    :titlesonly:
+    :glob:
 
+    autoapi/lib/sedna/index
+
+
+.. toctree::
+    :caption: ROADMAP
+    :hidden:
 
+    roadmap
 
 
+RELATED LINKS
+=============
 
+.. mdinclude:: related_link.md
 
 Indices and tables
 ==================

docs/lib-api/modules.rst

@@ -1,7 +0,0 @@
-sedna
-=======
-
-.. toctree::
-   :maxdepth: 4
-
-   sedna

docs/lib-api/sedna.hard_example_mining.image_classification.rst

@@ -1,16 +0,0 @@
-sedna.hard\_example\_mining.image\_classification package
-===========================================================
-
-.. automodule:: sedna.hard_example_mining.image_classification
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-sedna.hard\_example\_mining.image\_classification.hard\_mine\_filters module
-------------------------------------------------------------------------------
-
-.. automodule:: sedna.hard_example_mining.image_classification.hard_mine_filters
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/lib-api/sedna.hard_example_mining.object_detection.rst

@@ -1,18 +0,0 @@
-sedna.hard\_example\_mining.object\_detection package
-=======================================================
-
-.. automodule:: sedna.hard_example_mining.object_detection
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Submodules
-----------
-
-sedna.hard\_example\_mining.object\_detection.scores\_filters module
-----------------------------------------------------------------------
-
-.. automodule:: sedna.hard_example_mining.object_detection.scores_filters
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/lib-api/sedna.hard_example_mining.rst

@@ -1,27 +0,0 @@
-sedna.hard\_example\_mining package
-=====================================
-
-.. automodule:: sedna.hard_example_mining
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Subpackages
------------
-
-.. toctree::
-   :maxdepth: 4
-
-   sedna.hard_example_mining.image_classification
-   sedna.hard_example_mining.object_detection
-
-Submodules
-----------
-
-sedna.hard\_example\_mining.base module
------------------------------------------
-
-.. automodule:: sedna.hard_example_mining.base
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/lib-api/sedna.joint_inference.rst

@@ -1,26 +0,0 @@
-sedna.joint\_inference package
-================================
-
-.. automodule:: sedna.joint_inference
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Submodules
-----------
-
-sedna.joint\_inference.data module
-------------------------------------
-
-.. automodule:: sedna.joint_inference.data
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-sedna.joint\_inference.joint\_inference module
-------------------------------------------------
-
-.. automodule:: sedna.joint_inference.joint_inference
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/lib-api/sedna.rst

@@ -1,18 +0,0 @@
-
-Subpackages
------------
-
-.. toctree::
-   :maxdepth: 4
-
-   sedna.hard_example_mining
-   sedna.joint_inference
-
-sedna.context module
-----------------------
-
-.. automodule:: sedna.context
-   :members:
-   :undoc-members:
-   :show-inheritance:
-

docs/proposals/dataset-and-model.md

@@ -73,7 +73,7 @@ The tables below summarize the group, kind and API version details for the CRDs.
 
 #### `Dataset` CRD
 
-[crd source](/build/crds/sedna/dataset_v1alpha1.yaml)
+[crd source](/build/crds/sedna.io_datasets.yaml)
 
 ```yaml
 apiVersion: apiextensions.k8s.io/v1
@@ -144,7 +144,7 @@ Current we support these below formats:
 
 #### `Model` CRD
 
-[crd source](/build/crds/sedna/model_v1alpha1.yaml)
+[crd source](/build/crds/sedna.io_models.yaml)
 ```yaml
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -209,7 +209,7 @@ spec:
 ### CRD type definition
 - `Dataset`
 
-[go source](cloud/pkg/apis/sedna/v1alpha1/dataset_types.go)
+[go source](/pkg/apis/sedna/v1alpha1/dataset_types.go)
 
 ```go
 package v1alpha1
@@ -259,7 +259,7 @@ type DatasetList struct {
 
 - `Model`
 
-[go source](cloud/pkg/apis/sedna/v1alpha1/model_types.go)
+[go source](/pkg/apis/sedna/v1alpha1/model_types.go)
 ```go
 package v1alpha1
 

docs/proposals/incremental-learning.md

@@ -74,7 +74,7 @@ The tables below summarize the group, kind and API version details for the CRD.
 |Kind                   | IncrementalLearningJob             |
 
 ### Incremental learning CRD
-See the [crd source](/build/crds/sedna/incrementallearningjob_v1alpha1.yaml) for details.
+See the [crd source](/build/crds/sedna.io_incrementallearningjobs.yaml) for details.
 
 ### Incremental learning job type definition
 

docs/proposals/joint-inference.md

@@ -64,7 +64,7 @@ The tables below summarize the group, kind and API version details for the CRD.
 
 ### Joint inference CRD
 
-see [crd source](/build/crd-samples/sedna/jointinferenceservice_v1alpha1.yaml)
+see [crd source](/build/crds/sedna.io_jointinferenceservices.yaml)
 
 ### Joint inference type definition
 

docs/proposals/object-search.md

@@ -15,6 +15,7 @@
     - [Details of api between GM(cloud) and LC(edge)](#details-of-api-between-gmcloud-and-lcedge)
     - [Flow of object search service creation](#flow-of-object-search-service-creation)
   - [Workers Communication](#workers-communication)
+  
 # Object Search Service
 ## Motivation
 Object search is an important technology in the field of computer vision, which is widely used in security monitoring, intelligent transportation, etc. Generally, online object search applications have stringent latency constraints, which cannot be met by cloud computing schemes. Object search schemes based on edge computing have the characteristics of low latency and data privacy security, and are the mainstream technology trend. 

docs/proposals/object-tracking.md

@@ -15,6 +15,7 @@
     - [Details of api between GM(cloud) and LC(edge)](#details-of-api-between-gmcloud-and-lcedge)
     - [Flow of object tracking service creation](#flow-of-object-tracking-service-creation)
   - [Workers Communication](#workers-communication)
+  
 # Object Tracking Service
 ## Motivation
 Object tracking is an important technology in the field of computer vision, which is widely used in security monitoring, intelligent transportation, etc. Generally, object tracking applications have high latency requirements, which cannot be met by cloud computing schemes. The object tracking schemes based on edge computing have the characteristics of low latency and data privacy security, and are the mainstream technology trend. 

docs/quickstart.md

@@ -1,23 +1,50 @@
-## Getting start
 
-Sedna is an open source framework for edge-cloud collaborative training and inference, so that AI applications running at the edge can benefit from cost reduction, model performance improvement and data privacy protection.
+# Quick Start
+
+## Introduction
+
+Sedna provide some examples of running Sedna jobs in [here](/examples/README.md)
+
+Here is a general guide to quick start an incremental learning job.
 
 ### Get Sedna
 
-You can find the latest Sedna release [here](TODO)
+You can find the latest Sedna release [here](https://github.com/kubeedge/sedna/releases).
 
 ### Deploying Sedna
 
-Please refer to this [link](setup/install.html).
+Please refer to this [link](setup/install.md).
+
+### Prepare
+
+todo
+
+### Configure
+
+todo
+
+### Run
+
+todo
+
+### Monitor
+
+todo
+
+## API
+
+- CRD: Please refer to this [link](api/crd).
+- Lib: Please refer to this [link](api/lib).
 
-### Examples
-Please refer to this[link](TODO)
+## Contributing
 
-### Contributing
+Contributions are very welcome! You can see our [CONTRIBUTING](CONTRIBUTING.md) for more information.
 
-Contributions are very welcome! You can see our [CONTRIBUTING.md](TODO) for more information
+## Community
 
-### Community
+Sedna is an open source project and in the spirit of openness and freedom, we welcome new contributors to join us. 
+You can get in touch with the community according to the ways:
+* [Github Issues](https://github.com/kubeedge/sedna/issues)
+* [Regular Community Meeting](https://zoom.us/j/4167237304)
+* [slack channel](https://app.slack.com/client/TDZ5TGXQW/C01EG84REVB/details)
 
-Sedna is an open source project and In the spirit of openness and freedom, we welcome new contributors to join us . You can get in touch with the community according to the ways:
-* [Github Issues](TODO)

docs/related_link.md

@@ -0,0 +1 @@
+[支持边云协同终身学习特性，KubeEdge 子项目 Sedna 0.3.0 版本发布！](https://juejin.cn/post/6970866022286884878)

examples/README.md

@@ -6,6 +6,7 @@ This repository is home to the following features of examples:
 * [Joint Inference](#joint-inference)
 * [Incremental Learning](#incremental-learning)
 * [Federated Learning](#federated-learning)
+* [Lifelong Learning](#lifelong-learning)
 * [Shared Storage](#shared-storage)
 
 ### Joint Inference
@@ -17,6 +18,9 @@ Example: [Using Incremental Learning Job in Helmet Detection Scenario](./increme
 ### Federated Learning
 Example: [Using Federated Learning Job in Surface Defect Detection Scenario](./federated_learning/surface_defect_detection/README.md)
 
+### Lifelong Learning
+Example: [Using Lifelong Learning Job in Thermal Comfort Prediction Scenario](./lifelong_learning/atcii/README.md)
+
 ### Shared Storage
 | Support Protocols  |Support Features| Examples |Release|
 | :-------------: | :-------------: |:-------------: | :-------------: |