fixed robo skills

Signed-off-by: SiqiLuo <1587295470@qq.com>

.gitignore

@@ -28,3 +28,8 @@ __pycache__/
 
 # go build output
 /_output
+
+# AI model files
+*.pth
+*.model
+*.pkl

docs/contributing/organization_structure.md

@@ -0,0 +1,22 @@
+Following the [KubeEdge Community Membership](https://github.com/kubeedge/community/blob/master/community-membership.md#member), we propose the membership arrangement for Kubeedge SIG AI.
+
+# Kubeedge SIG AI Membership
+| Role | Member List |
+| ----- | ------------------ |
+| Contributors | [Contributor List](https://github.com/kubeedge/sedna/graphs/contributors) |
+| Members | 
+| Reviewers | [jaypume](https://github.com/jaypume) <br> [Poorunga](https://github.com/Poorunga) <br> [JimmyYang20](https://github.com/JimmyYang20) <br> [TymonXie](https://github.com/TymonXie) <br> [llhuii](https://github.com/llhuii) <br> [JoeyHwong-gk](https://github.com/JoeyHwong-gk)|
+| Approvers | None|
+| Maintainers | None |
+| Owners | None |
+
+
+# Kubeedge SIG AI Owners of Features
+|  | Features | Owners | Other collaborative members
+| ---- |----- | ------------------ | ------- |
+| Sedna | AI Framework | [jaypume](https://github.com/jaypume) |[JoeyHwong-gk](https://github.com/JoeyHwong-gk) |
+| | Joint inference | [jaypume](https://github.com/jaypume) | |
+| | Incremental training | [jaypume](https://github.com/jaypume) | |
+| | Federated learning | [jaypume](https://github.com/jaypume) | |
+| | Lifelong learning | [MooreZheng](https://github.com/MooreZheng) | [luosiqi](https://github.com/luosiqi)|
+

examples/lifelong-learning-robo-rfnet.Dockerfile

@@ -10,10 +10,10 @@ COPY ./lib/requirements.dev.txt /home
 # install requirements of sedna lib
 RUN pip install -r /home/requirements.txt
 RUN pip install -r /home/requirements.dev.txt
-RUN pip install joblib
+RUN pip install joblib~=1.2.0
 RUN pip install pandas
 RUN pip install scikit-learn~=0.23.2
-RUN pip install torchvision
+RUN pip install torchvision~=0.13.0
 RUN pip install Pillow
 RUN pip install tqdm
 RUN pip install minio
@@ -22,13 +22,14 @@ RUN pip install matplotlib
 RUN pip install opencv-python
 RUN pip install python-multipart
 RUN pip install tensorboard
+RUN pip install watchdog
 
 ENV PYTHONPATH "/home/lib"
 
 WORKDIR /home/work
 COPY ./lib /home/lib
 
-COPY examples/lifelong_learning/robot_dog_delivery  /home/work/
-WORKDIR /home/work/robot_dog_delivery/RFNet
+COPY ./examples/lifelong_learning/robot_dog_delivery  /home/work/
+WORKDIR /home/work/RFNet
 
 ENTRYPOINT ["python"]

examples/lifelong_learning/robot_dog_delivery/RFNet/basemodel.py

@@ -4,7 +4,6 @@ import cv2
 import numpy as np
 import torch
 from PIL import Image
-
 from torchvision import transforms
 from torch.utils.data import DataLoader
 from torchvision import transforms
@@ -101,10 +100,10 @@ class Model:
         self.val_args.weight_path = kwargs.get("weight_path")
         self.validator = Validator(self.val_args)
 
-        self.ramp_val_args = EvaluationArguments()
-        self.ramp_val_args.weight_path = "/home/lsq/RFNet/models/ramp_train1_200.pth"
-        self.ramp_val_args.merge = False
-        self.validator_ramp = Validator(self.ramp_val_args)
+        # self.ramp_val_args = EvaluationArguments()
+        # self.ramp_val_args.weight_path = "/home/lsq/RFNet/models/ramp_train1_200.pth"
+        # self.ramp_val_args.merge = False
+        # self.validator_ramp = Validator(self.ramp_val_args)
 
     def train(self, train_data, valid_data=None, **kwargs):
         self.trainer = Trainer(self.train_args, train_data=train_data)
@@ -139,49 +138,50 @@ class Model:
         self.train_model_url = train_model_url
         return self.train_model_url
 
-    # def predict(self, data, **kwargs):
-    #     prediction = kwargs.get('prediction')
-    #     if isinstance(data[0], dict):
-    #         data = preprocess_frames(data)
-    #
-    #     if isinstance(data[0], np.ndarray):
-    #         data = preprocess_url(data)
-    #
-    #     self.validator.test_loader = DataLoader(
-    #         data,
-    #         batch_size=self.val_args.test_batch_size,
-    #         shuffle=False,
-    #         pin_memory=False)
-    #     if not prediction:
-    #         return self.validator.validate()
-    #     else:
-    #         return prediction
-
     def predict(self, data, **kwargs):
-        if isinstance(data[0], np.ndarray):
-            data = preprocess_url(data)
-
+        prediction = kwargs.get('prediction')
         if isinstance(data[0], dict):
             data = preprocess_frames(data)
-
+    
+        if isinstance(data[0], np.ndarray):
+            data = preprocess_url(data)
+    
         self.validator.test_loader = DataLoader(
             data,
             batch_size=self.val_args.test_batch_size,
             shuffle=False,
             pin_memory=False)
 
-        # TODO: predict ramp using specific model
-        self.validator_ramp.test_loader = DataLoader(
-            data,
-            batch_size=self.val_args.test_batch_size,
-            shuffle=False,
-            pin_memory=False)
-
-        prediction = kwargs.get('prediction')
         if not prediction:
-            return (self.validator.validate(), self.validator_ramp.validate())
+            return self.validator.validate()
         else:
-            return (prediction, self.validator_ramp.validate())
+            return prediction
+
+    # def predict(self, data, **kwargs):
+    #     if isinstance(data[0], np.ndarray):
+    #         data = preprocess_url(data)
+
+    #     if isinstance(data[0], dict):
+    #         data = preprocess_frames(data)
+
+    #     self.validator.test_loader = DataLoader(
+    #         data,
+    #         batch_size=self.val_args.test_batch_size,
+    #         shuffle=False,
+    #         pin_memory=False)
+
+    #     # TODO: predict ramp using specific model
+    #     self.validator_ramp.test_loader = DataLoader(
+    #         data,
+    #         batch_size=self.val_args.test_batch_size,
+    #         shuffle=False,
+    #         pin_memory=False)
+
+    #     prediction = kwargs.get('prediction')
+    #     if not prediction:
+    #         return (self.validator.validate(), self.validator_ramp.validate())
+    #     else:
+    #         return (prediction, self.validator_ramp.validate())
 
     def evaluate(self, data, **kwargs):
         predictions = self.predict(data.x)

examples/lifelong_learning/robot_dog_delivery/RFNet/evaluate.py

@@ -1,10 +1,8 @@
 import os
 os.environ["TEST_DATASET_URL"] = "/home/lsq/RFNet/data_index/test.txt"
 os.environ["MODEL_URLS"] = "s3://kubeedge/sedna-robo/kb/index.pkl"
-os.environ["OUTPUT_URL"] = "s3://kubeedge/sedna-robo/kb/"
-os.environ["S3_ENDPOINT_URL"] = "https://obs.cn-north-1.myhuaweicloud.com"
-os.environ["SECRET_ACCESS_KEY"] = "OYPxi4uD9k5E90z0Od3Ug99symbJZ0AfyB4oveQc"
-os.environ["ACCESS_KEY_ID"] = "EMPTKHQUGPO2CDUFD2YR"
+os.environ["OUTPUT_URL"] = "s3://kubeedge/sedna-robo/kb_next/"
+
 os.environ["KB_SERVER"] = "http://0.0.0.0:9020"
 os.environ["operator"] = "<"
 os.environ["model_threshold"] = "0.01"
@@ -13,7 +11,7 @@ from sedna.core.lifelong_learning import LifelongLearning
 from sedna.datasources import TxtDataParse
 from sedna.common.config import Context
 
-from accuracy import robo_accuracy
+from accuracy import accuracy
 from basemodel import Model
 
 
@@ -38,23 +36,19 @@ def eval():
         "method": "TaskAllocationSimple"
     }
 
-    inference_integrate = {
-        "method": "InferenceIntegrateByType"
-    }
-
     ll_job = LifelongLearning(estimator,
                               task_definition=None,
                               task_relationship_discovery=None,
                               task_allocation=task_allocation,
                               task_remodeling=None,
-                              inference_integrate=inference_integrate,
+                              inference_integrate=None,
                               task_update_decision=None,
                               unseen_task_allocation=None,
                               unseen_sample_recognition=None,
                               unseen_sample_re_recognition=None
                               )
 
-    ll_job.evaluate(eval_data, metrics=robo_accuracy)
+    ll_job.evaluate(eval_data, metrics=accuracy)
 
 
 if __name__ == '__main__':

examples/lifelong_learning/robot_dog_delivery/RFNet/predict.py

@@ -1,22 +1,22 @@
 import os
-os.environ["MODEL_URLS"] = "s3://kubeedge/sedna-robo/kb/index.pkl"
-os.environ["S3_ENDPOINT_URL"] = "https://obs.cn-north-1.myhuaweicloud.com"
-os.environ["SECRET_ACCESS_KEY"] = "OYPxi4uD9k5E90z0Od3Ug99symbJZ0AfyB4oveQc"
-os.environ["ACCESS_KEY_ID"] = "EMPTKHQUGPO2CDUFD2YR"
-os.environ["KB_SERVER"] = "http://0.0.0.0:9020"
+import time
+os.environ["MODEL_URLS"] = "s3://kubeedge/sedna-robo/kb_next/index.pkl"
+
+# set in yaml
 os.environ["unseen_save_url"] = "s3://kubeedge/sedna-robo/unseen_samples/"
+os.environ["OOD_model"] = "s3://kubeedge/sedna-robo/models/lr_model.model"
+os.environ["OOD_thresh"] = "0.0001"
 
-import cv2
-import time
-import numpy as np
-from PIL import Image
-import warnings
+os.environ["robo_skill"] = "ramp_detection"
+os.environ["ramp_detection"] = "s3://kubeedge/sedna-robo/models/ramp_train1_200.pth"
+os.environ["curb_detection"] = "s3://kubeedge/sedna-robo/models/2048x1024_80.pth"
 
-from sedna.datasources import BaseDataSource, TxtDataParse
-from basemodel import Model, preprocess_frames
+from sedna.datasources import BaseDataSource
 from sedna.core.lifelong_learning import LifelongLearning
 from sedna.common.config import Context
 
+from basemodel import Model, preprocess_frames
+
 
 def preprocess(samples):
     data = BaseDataSource(data_type="test")
@@ -34,20 +34,10 @@ def postprocess(samples):
     return image_names, imgs
 
 
-def _load_txt_dataset(dataset_url):
-    # use original dataset url
-    original_dataset_url = Context.get_parameters('original_dataset_url', "")
-    dataset_urls = dataset_url.split()
-    dataset_urls = [
-        os.path.join(
-            os.path.dirname(original_dataset_url),
-            dataset_url) for dataset_url in dataset_urls]
-    return dataset_urls[:-1], dataset_urls[-1]
-
-
-def init_ll_job(**kwargs):
+def init_ll_job():
+    robo_skill = Context.get_parameters("robo_skill", "ramp_detection")
     estimator = Model(num_class=31,
-                      weight_path=kwargs.get('weight_path'),
+                      weight_path=Context.get_parameters(robo_skill),
                       save_predicted_image=True,
                       merge=True)
 
@@ -57,26 +47,17 @@ def init_ll_job(**kwargs):
     unseen_task_allocation = {
         "method": "UnseenTaskAllocationDefault"
     }
-
     unseen_sample_recognition = {
         "method": "OodIdentification",
         "param": {
-            "OOD_thresh": float(kwargs.get("OOD_thresh")),
-            "backup_model": kwargs.get("OOD_backup_model"),
-            "OOD_model_path": kwargs.get("OOD_model"),
+            "OOD_thresh": float(Context.get_parameters("OOD_thresh")),
+            "OOD_model": Context.get_parameters("OOD_model"),
+            "OOD_backup_model": Context.get_parameters(robo_skill),
             "preprocess_func": preprocess_frames,
             "base_model": Model
         }
     }
 
-    # unseen_sample_recognition = {
-    #     "method": "SampleRegonitionRobotic"
-    # }
-
-    inference_integrate = {
-        "method": "InferenceIntegrateByType"
-    }
-
     ll_job = LifelongLearning(
         estimator,
         unseen_estimator=unseen_task_processing,
@@ -84,62 +65,13 @@ def init_ll_job(**kwargs):
         task_relationship_discovery=None,
         task_allocation=task_allocation,
         task_remodeling=None,
-        inference_integrate=inference_integrate,
+        inference_integrate=None,
         task_update_decision=None,
         unseen_task_allocation=unseen_task_allocation,
         unseen_sample_recognition=unseen_sample_recognition,
         unseen_sample_re_recognition=None)
-
     return ll_job
 
 
 def unseen_task_processing():
     return "Warning: unseen sample detected."
-
-
-def predict():
-    ll_job = init_ll_job()
-
-    camera_address = Context.get_parameters('video_url')
-    # use video streams for testing
-    camera = cv2.VideoCapture(camera_address)
-    fps = 10
-    nframe = 0
-    while 1:
-        ret, input_yuv = camera.read()
-        if not ret:
-            time.sleep(5)
-            camera = cv2.VideoCapture(camera_address)
-            continue
-
-        if nframe % fps:
-            nframe += 1
-            continue
-
-        img_rgb = cv2.cvtColor(input_yuv, cv2.COLOR_BGR2RGB)
-        nframe += 1
-        if nframe % 1000 == 1:  # logs every 1000 frames
-            warnings.warn(f"camera is open, current frame index is {nframe}")
-
-        img_rgb = cv2.resize(np.array(img_rgb), (2048, 1024),
-                             interpolation=cv2.INTER_CUBIC)
-        img_rgb = Image.fromarray(img_rgb)
-        data = {'image': img_rgb, "depth": img_rgb, "label": img_rgb}
-        data = preprocess(data)
-        print(postprocess)
-        print("Inference results:", ll_job.inference(
-            data=data, post_process=postprocess))
-
-
-def predict_batch():
-    ll_job = init_ll_job()
-
-    test_dataset_url = Context.get_parameters("test_dataset_url")
-    test_data = TxtDataParse(data_type="test", func=_load_txt_dataset)
-    test_data.parse(test_dataset_url, use_raw=False)
-
-    return ll_job.inference(data=test_data)
-
-
-if __name__ == '__main__':
-    print(predict())