Browse Source

Fix downstream bug in IL

bug: after deleting dataset, the event of deleting job can not be sent to LC.

Signed-off-by: JimmyYang20 <yangjin39@huawei.com>
tags/v0.5.0
JimmyYang20 3 years ago
parent
commit
700d1697dd
1 changed files with 14 additions and 8 deletions
  1. +14
    -8
      pkg/globalmanager/controllers/incrementallearning/downstream.go

+ 14
- 8
pkg/globalmanager/controllers/incrementallearning/downstream.go View File

@@ -62,18 +62,15 @@ func (c *Controller) syncToEdge(eventType watch.EventType, obj interface{}) erro
// more details at https://github.com/kubernetes/kubernetes/issues/3030
job.Kind = KindName

jobConditions := job.Status.Conditions
if len(jobConditions) == 0 {
return nil
}

dataName := job.Spec.Dataset.Name
// LC has dataset object on this node that may call dataset node
var dsNodeName string
ds, err := c.client.Datasets(job.Namespace).Get(context.TODO(), dataName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("dataset(%s/%s) not found", job.Namespace, dataName)
klog.Errorf("not found job(name=%s/%s)'s dataset, error: %v", job.Kind, job.Name, err)
} else {
dsNodeName = ds.Spec.NodeName
}
// LC has dataset object on this node that may call dataset node
dsNodeName := ds.Spec.NodeName

var trainNodeName string
var evalNodeName string
@@ -102,6 +99,15 @@ func (c *Controller) syncToEdge(eventType watch.EventType, obj interface{}) erro
return nil
}

if dsNodeName == "" {
return nil
}

jobConditions := job.Status.Conditions
if len(jobConditions) == 0 {
return nil
}

latestCondition := jobConditions[len(jobConditions)-1]
currentType := latestCondition.Type
jobStage := latestCondition.Stage


Loading…
Cancel
Save