Browse Source

Fix the review comments

Signed-off-by: JimmyYang20 <yangjin39@huawei.com>
tags/v0.3.0
JimmyYang20 JimmyYang20 4 years ago
parent
commit
babb8f9bde
5 changed files with 12 additions and 16 deletions
  1. +0
    -2
      build/gm/gm-config.yaml
  2. +7
    -12
      pkg/globalmanager/lifelonglearningjob.go
  3. +1
    -1
      pkg/globalmanager/upstream.go
  4. +3
    -0
      pkg/localcontroller/manager/dataset.go
  5. +1
    -1
      pkg/localcontroller/util/util.go

+ 0
- 2
build/gm/gm-config.yaml View File

@@ -6,5 +6,3 @@ websocket:
port: 9000 port: 9000
localController: localController:
server: http://localhost:9100 server: http://localhost:9100
knowledgeBaseService:
server: http://localhost:9020

+ 7
- 12
pkg/globalmanager/lifelonglearningjob.go View File

@@ -57,7 +57,6 @@ var ljControllerKind = sednav1.SchemeGroupVersion.WithKind("LifelongLearningJob"
type LifelongLearningJobController struct { type LifelongLearningJobController struct {
kubeClient kubernetes.Interface kubeClient kubernetes.Interface
client sednaclientset.SednaV1alpha1Interface client sednaclientset.SednaV1alpha1Interface
podControl k8scontroller.PodControlInterface


// podStoreSynced returns true if the pod store has been synced at least once. // podStoreSynced returns true if the pod store has been synced at least once.
// Added as a member to the struct to allow injection for testing. // Added as a member to the struct to allow injection for testing.
@@ -273,7 +272,7 @@ func (jc *LifelongLearningJobController) sync(key string) (bool, error) {
jobFailed := false jobFailed := false
needUpdated := false needUpdated := false


// update conditions of incremental job
// update conditions of lifelonglearning job
needUpdated, err = jc.updateLifelongLearningJobConditions(&lifelonglearningjob) needUpdated, err = jc.updateLifelongLearningJobConditions(&lifelonglearningjob)
if err != nil { if err != nil {
klog.V(2).Infof("lifelonglearning job %v/%v faied to be updated, err:%s", lifelonglearningjob.Namespace, lifelonglearningjob.Name, err) klog.V(2).Infof("lifelonglearning job %v/%v faied to be updated, err:%s", lifelonglearningjob.Namespace, lifelonglearningjob.Name, err)
@@ -599,6 +598,7 @@ func (jc *LifelongLearningJobController) createPod(job *sednav1.LifelongLearning
"WORKER_NAME": "eval-worker-" + utilrand.String(5), "WORKER_NAME": "eval-worker-" + utilrand.String(5),


"LC_SERVER": jc.cfg.LC.Server, "LC_SERVER": jc.cfg.LC.Server,
"KB_SERVER": jc.cfg.KB.Server,
} }


var modelMountURLs []MountURL var modelMountURLs []MountURL
@@ -676,8 +676,8 @@ func (jc *LifelongLearningJobController) createInferPod(job *sednav1.LifelongLea
URL: inferModelURL, URL: inferModelURL,
Secret: jobSecret, Secret: jobSecret,
}, },
Name: "model",
EnvName: "MODEL_URL",
Name: "models",
EnvName: "MODEL_URLS",
}, },
) )


@@ -736,14 +736,9 @@ func NewLifelongLearningJobController(cfg *config.ControllerConfig) (FeatureCont
jc := &LifelongLearningJobController{ jc := &LifelongLearningJobController{
kubeClient: kubeClient, kubeClient: kubeClient,
client: crdclient.SednaV1alpha1(), client: crdclient.SednaV1alpha1(),
podControl: k8scontroller.RealPodControl{
KubeClient: kubeClient,
Recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "lifelonglearningjob-controller"}),
},

queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(DefaultBackOff, MaxBackOff), "lifelonglearningjob"),
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "lifelonglearningjob-controller"}),
cfg: cfg,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(DefaultBackOff, MaxBackOff), "lifelonglearningjob"),
recorder: eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "lifelonglearningjob-controller"}),
cfg: cfg,
} }


jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{


+ 1
- 1
pkg/globalmanager/upstream.go View File

@@ -384,7 +384,7 @@ func (uc *UpstreamController) appendLifelongLearningJobStatusCondition(name, nam
}) })
} }


// updateIncrementalLearningFromEdge syncs the edge updates to k8s
// updateLifelongLearningJobFromEdge syncs the edge updates to k8s
func (uc *UpstreamController) updateLifelongLearningJobFromEdge(name, namespace, operation string, content []byte) error { func (uc *UpstreamController) updateLifelongLearningJobFromEdge(name, namespace, operation string, content []byte) error {
err := checkUpstreamOperation(operation) err := checkUpstreamOperation(operation)
if err != nil { if err != nil {


+ 3
- 0
pkg/localcontroller/manager/dataset.go View File

@@ -239,6 +239,9 @@ func (ds *Dataset) readByLine(url string, format string) (*DataSource, error) {
numberOfSamples += len(samples) numberOfSamples += len(samples)
case DatasetFormatCSV: case DatasetFormatCSV:
// the first row of csv file is header // the first row of csv file is header
if len(samples) == 0 {
return nil, fmt.Errorf("file %s is empty", url)
}
dataSource.Header = samples[0] dataSource.Header = samples[0]
samples = samples[1:] samples = samples[1:]
numberOfSamples += len(samples) numberOfSamples += len(samples)


+ 1
- 1
pkg/localcontroller/util/util.go View File

@@ -54,7 +54,7 @@ func IsDir(path string) bool {
} }


// CopyFile copies a file to other // CopyFile copies a file to other
func CopyFile(dstName, srcName string) (written int64, err error) {
func CopyFile(srcName, dstName string) (written int64, err error) {
src, err := os.Open(srcName) src, err := os.Open(srcName)
if err != nil { if err != nil {
klog.Errorf("open file %s failed: %v", srcName, err) klog.Errorf("open file %s failed: %v", srcName, err)


Loading…
Cancel
Save