Skip existing items in RefreshStoreVectors()

object/vector.go

@@ -72,6 +72,20 @@ func getVector(owner string, name string) (*Vector, error) {
 	}
 }
 
+func getVectorByIndex(owner string, store string, file string, index int) (*Vector, error) {
+	vector := Vector{Owner: owner, Store: store, File: file, Index: index}
+	existed, err := adapter.engine.Get(&vector)
+	if err != nil {
+		return &vector, err
+	}
+
+	if existed {
+		return &vector, nil
+	} else {
+		return nil, nil
+	}
+}
+
 func GetVector(id string) (*Vector, error) {
 	owner, name := util.GetOwnerAndNameFromId(id)
 	return getVector(owner, name)

object/vector_embedding.go

@@ -93,6 +93,17 @@ func addVectorsForStore(storageProviderObj storage.StorageProvider, embeddingPro
 
 		textSections := txt.GetTextSections(text)
 		for i, textSection := range textSections {
+			var vector *Vector
+			vector, err = getVectorByIndex("admin", storeName, file.Key, i)
+			if err != nil {
+				return false, err
+			}
+
+			if vector != nil {
+				fmt.Printf("[%d/%d] Generating embedding for store: [%s]'s text section: %s\n", i+1, len(textSections), storeName, "Skipped due to already exists")
+				continue
+			}
+
 			if timeLimiter.Allow() {
 				fmt.Printf("[%d/%d] Generating embedding for store: [%s]'s text section: %s\n", i+1, len(textSections), storeName, textSection)
 				affected, err = addEmbeddedVector(embeddingProviderObj, textSection, storeName, file.Key, i, embeddingProviderName)