增加数据集的私有搜索

Signed-off-by: zouap <zouap@pcl.ac.cn>

models/dbsql/dataset_foreigntable_for_es.sql

@@ -55,6 +55,9 @@ DELETE FROM public.dataset_es;
         b.updated_unix,(select array_to_string(array_agg(name order by created_unix desc),'-#,#-') from public.attachment a where a.dataset_id=b.id and a.is_private=false)
         FROM public.dataset b,public.repository c where b.repo_id=c.id and c.is_private=false;
 
+
+DROP TRIGGER  IF EXISTS es_insert_dataset on public.dataset;
+
 CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS 
 $def$
     DECLARE
@@ -97,7 +100,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER  IF EXISTS es_insert_dataset on public.dataset;
+
 
 CREATE TRIGGER es_insert_dataset
     AFTER INSERT ON public.dataset
@@ -105,6 +108,9 @@ CREATE TRIGGER es_insert_dataset
 
 ALTER TABLE public.dataset ENABLE ALWAYS TRIGGER es_insert_dataset;
 
+
+DROP TRIGGER IF EXISTS es_udpate_dataset_file_name_delete on public.attachment;
+
 CREATE OR REPLACE FUNCTION public.udpate_dataset_file_name_delete() RETURNS trigger AS 
 $def$
     BEGIN
@@ -114,7 +120,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER IF EXISTS es_udpate_dataset_file_name_delete on public.attachment;
+
 CREATE TRIGGER es_udpate_dataset_file_name_delete
     AFTER DELETE ON public.attachment
     FOR EACH ROW EXECUTE PROCEDURE udpate_dataset_file_name_delete();
@@ -145,6 +151,8 @@ CREATE TRIGGER es_update_dataset
 
 ALTER TABLE public.dataset ENABLE ALWAYS TRIGGER es_update_dataset;
 
+DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset;
+
 CREATE OR REPLACE FUNCTION public.delete_dataset() RETURNS trigger AS 
 $def$
     declare
@@ -155,7 +163,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset;
+
 CREATE TRIGGER es_delete_dataset
     AFTER DELETE ON public.dataset
     FOR EACH ROW EXECUTE PROCEDURE delete_dataset();

models/dbsql/repo_foreigntable_for_es.sql

@@ -156,6 +156,7 @@ delete from  public.repository_es;
                                         lower_alias
                                         FROM  public.repository b where b.is_private=false;
 
+DROP TRIGGER  IF EXISTS es_insert_repository on public.repository;
 
 CREATE OR REPLACE FUNCTION public.insert_repository_data() RETURNS trigger AS 
 $def$
@@ -257,8 +258,6 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER  IF EXISTS es_insert_repository on public.repository;
-
 
 CREATE TRIGGER es_insert_repository
     AFTER INSERT ON public.repository
@@ -483,6 +482,9 @@ CREATE TRIGGER es_update_repository
 
 ALTER TABLE public.repository ENABLE ALWAYS TRIGGER es_update_repository;
 
+
+DROP TRIGGER IF EXISTS es_delete_repository on public.repository;
+
 CREATE OR REPLACE FUNCTION public.delete_repository() RETURNS trigger AS 
 $def$
     declare
@@ -495,7 +497,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER IF EXISTS es_delete_repository on public.repository;
+
 CREATE TRIGGER es_delete_repository
     AFTER DELETE ON public.repository
     FOR EACH ROW EXECUTE PROCEDURE delete_repository();
@@ -514,7 +516,7 @@ $def$
 	   elsif (TG_OP = 'INSERT') then
 	       update public.repository_es  SET lang=(select  array_to_string(array_agg(language order by percentage desc),',')  from public.language_stat where repo_id=NEW.repo_id) where id=NEW.repo_id;
        elsif (TG_OP = 'DELETE') then
-           if exists(select 1 from public.repository where id=OLD.repo_id)
+           if exists(select 1 from public.repository where id=OLD.repo_id) then
 	           update public.repository_es  SET lang=(select  array_to_string(array_agg(language order by percentage desc),',')  from public.language_stat where repo_id=OLD.repo_id) where id=OLD.repo_id;
 	       end if;
        end if;

models/dbsql/user_foreigntable_for_es.sql

@@ -49,7 +49,8 @@ CREATE FOREIGN TABLE public.user_es
     token character varying(1024) ,
     public_key character varying(255),
     private_key character varying(255),
-    is_operator boolean NOT NULL DEFAULT false
+    is_operator boolean NOT NULL DEFAULT false,
+    num_dataset_stars integer NOT NULL DEFAULT 0
 ) SERVER multicorn_es
 OPTIONS
     (
@@ -103,7 +104,8 @@ delete from public.user_es;
         repo_admin_change_team_access, 
         diff_view_style, 
         theme, 
-        is_operator)
+        is_operator,
+        num_dataset_stars)
 	    SELECT 
         id, 
         lower_name,
@@ -146,9 +148,12 @@ delete from public.user_es;
         repo_admin_change_team_access, 
         diff_view_style, 
         theme, 
-        is_operator
+        is_operator,
+        num_dataset_stars
         FROM public.user;
 
+DROP TRIGGER  IF EXISTS es_insert_user on public.user;
+
 CREATE OR REPLACE FUNCTION public.insert_user_data() RETURNS trigger AS 
 $def$
     BEGIN
@@ -194,7 +199,8 @@ $def$
         repo_admin_change_team_access, 
         diff_view_style, 
         theme, 
-        is_operator)
+        is_operator,
+        num_dataset_stars)
 	    VALUES (
         NEW.id, 
         NEW.lower_name,
@@ -237,7 +243,8 @@ $def$
         NEW.repo_admin_change_team_access, 
         NEW.diff_view_style, 
         NEW.theme, 
-        NEW.is_operator
+        NEW.is_operator,
+        NEW.num_dataset_stars
         );
 
         RETURN NEW;
@@ -245,7 +252,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER  IF EXISTS es_insert_user on public.user;
+
 
 CREATE TRIGGER es_insert_user
     AFTER INSERT ON public.user
@@ -253,6 +260,7 @@ CREATE TRIGGER es_insert_user
 
 ALTER TABLE public.user ENABLE ALWAYS TRIGGER es_insert_user;
 
+DROP TRIGGER  IF EXISTS es_update_user on public.user;
 
 CREATE OR REPLACE FUNCTION public.update_user() RETURNS trigger AS 
 $def$
@@ -263,14 +271,16 @@ $def$
        full_name=NEW.full_name,
        location=NEW.location,
        website=NEW.website,
-       email=NEW.email
+       email=NEW.email,
+       num_dataset_stars=NEW.num_dataset_stars,
+       updated_unix=NEW.updated_unix
        where id=NEW.id;
        return new;
     END
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER  IF EXISTS es_update_user on public.user;
+
 
 CREATE TRIGGER es_update_user
     AFTER UPDATE  ON public.user
@@ -278,6 +288,8 @@ CREATE TRIGGER es_update_user
 
 ALTER TABLE public.user ENABLE ALWAYS TRIGGER es_update_user;
 
+DROP TRIGGER IF EXISTS es_delete_user on public.user;
+
 CREATE OR REPLACE FUNCTION public.delete_user() RETURNS trigger AS 
 $def$
     declare
@@ -288,7 +300,7 @@ $def$
 $def$ 
 LANGUAGE plpgsql;
 
-DROP TRIGGER IF EXISTS es_delete_user on public.user;
+
 CREATE TRIGGER es_delete_user
     AFTER DELETE ON public.user
     FOR EACH ROW EXECUTE PROCEDURE delete_user();

models/search_record.go

@@ -28,7 +28,7 @@ func SaveSearchKeywordToDb(keyword string) error {
 	return nil
 }
 
-func setQueryCondition(sess *xorm.Session, Keyword string, isPull bool, userId int64) {
+func setIssueQueryCondition(sess *xorm.Session, Keyword string, isPull bool, userId int64) {
 	sess.And("issue.poster_id=?", userId)
 	sess.And("issue.is_pull=?", isPull)
 	sess.And("(issue.name like '%" + Keyword + "%' or issue.content like '%" + Keyword + "%')")
@@ -38,13 +38,13 @@ func setQueryCondition(sess *xorm.Session, Keyword string, isPull bool, userId i
 func SearchPrivateIssueOrPr(Page int, PageSize int, Keyword string, isPull bool, userId int64) ([]*Issue, int64, error) {
 	sess := x.NewSession()
 	defer sess.Close()
-	setQueryCondition(sess, Keyword, isPull, userId)
+	setIssueQueryCondition(sess, Keyword, isPull, userId)
 	count, err := sess.Count(new(Issue))
 	if err != nil {
 		return nil, 0, err
 	}
 
-	setQueryCondition(sess, Keyword, isPull, userId)
+	setIssueQueryCondition(sess, Keyword, isPull, userId)
 	sess.Desc("issue.created_unix")
 	sess.Limit(PageSize, (Page-1)*PageSize)
 	issues := make([]*Issue, 0)
@@ -53,5 +53,31 @@ func SearchPrivateIssueOrPr(Page int, PageSize int, Keyword string, isPull bool,
 	} else {
 		return issues, count, nil
 	}
+}
+
+func setDataSetQueryCondition(sess *xorm.Session, Keyword string, userId int64) {
+	sess.And("dataset.user_id=?", userId)
+	sess.And("(dataset.title like '%" + Keyword + "%' or dataset.description like '%" + Keyword + "%')")
+	sess.Join("INNER", "repository", "dataset.repo_id = repository.id").And("repository.is_private = ?", true)
+}
+
+func SearchDatasetBySQL(Page int, PageSize int, Keyword string, userId int64) ([]*Dataset, int64, error) {
+	sess := x.NewSession()
+	defer sess.Close()
+	setDataSetQueryCondition(sess, Keyword, userId)
+	count, err := sess.Count(new(Dataset))
+	if err != nil {
+		return nil, 0, err
+	}
+
+	setDataSetQueryCondition(sess, Keyword, userId)
+	sess.Desc("issue.created_unix")
+	sess.Limit(PageSize, (Page-1)*PageSize)
+	datasets := make([]*Dataset, 0)
+	if err := sess.Find(&datasets); err != nil {
+		return nil, 0, err
+	} else {
+		return datasets, count, nil
+	}
 
 }

routers/search.go

@@ -770,9 +770,54 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int,
 	   download_times
 
 	*/
+	log.Info("query searchdataset start")
 	SortBy := ctx.Query("SortBy")
 	ascending := ctx.QueryBool("Ascending")
-	log.Info("query searchRepo start")
+	PrivateTotal := ctx.QueryInt("PrivateTotal")
+	WebTotal := ctx.QueryInt("WebTotal")
+
+	from := (Page - 1) * PageSize
+	if from == 0 {
+		WebTotal = 0
+	}
+	resultObj := &SearchRes{}
+	log.Info("WebTotal=" + fmt.Sprint(WebTotal))
+	log.Info("PrivateTotal=" + fmt.Sprint(PrivateTotal))
+	resultObj.Result = make([]map[string]interface{}, 0)
+
+	if ctx.User != nil && (from < PrivateTotal || from == 0) {
+
+		log.Info("actor is null?:" + fmt.Sprint(ctx.User == nil))
+		datasets, count, err := models.SearchDatasetBySQL(Page, PageSize, Key, ctx.User.ID)
+		if err != nil {
+			ctx.JSON(200, "")
+			return
+		}
+		resultObj.PrivateTotal = count
+		datasetSize := len(datasets)
+		if datasetSize > 0 {
+			log.Info("Query private dataset number is:" + fmt.Sprint(datasetSize) + " count=" + fmt.Sprint(count))
+			makePrivateDataSet(datasets, resultObj, Key)
+		} else {
+			log.Info("not found private dataset, keyword=" + Key)
+		}
+		if datasetSize >= PageSize {
+			if WebTotal > 0 { //next page， not first query.
+				resultObj.Total = int64(WebTotal)
+				ctx.JSON(200, resultObj)
+				return
+			}
+		}
+	} else {
+		resultObj.PrivateTotal = int64(PrivateTotal)
+	}
+
+	from = from - PrivateTotal
+	if from < 0 {
+		from = 0
+	}
+	Size := PageSize - len(resultObj.Result)
+
 	boolQ := elastic.NewBoolQuery()
 	if Key != "" {
 		nameQuery := elastic.NewMatchQuery("title", Key).Boost(2).QueryName("f_first")
@@ -780,24 +825,30 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int,
 		fileNameQuery := elastic.NewMatchQuery("file_name", Key).Boost(1).QueryName("f_third")
 		categoryQuery := elastic.NewMatchQuery("category", Key).Boost(1).QueryName("f_fourth")
 		boolQ.Should(nameQuery, descQuery, categoryQuery, fileNameQuery)
-		res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context())
+		res, err := client.Search(TableName).Query(boolQ).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Highlight(queryHighlight("title", "description", "file_name", "category")).Do(ctx.Req.Context())
 		if err == nil {
 			searchJson, _ := json.Marshal(res)
 			log.Info("searchJson=" + string(searchJson))
-			result := makeDatasetResult(res, Key, OnlyReturnNum)
-			ctx.JSON(200, result)
+			esresult := makeDatasetResult(res, Key, OnlyReturnNum)
+			resultObj.Total = resultObj.PrivateTotal + esresult.Total
+			log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total))
+			resultObj.Result = append(resultObj.Result, esresult.Result...)
+			ctx.JSON(200, resultObj)
 		} else {
 			log.Info("query es error," + err.Error())
 		}
 	} else {
-		log.Info("query all content.")
+		log.Info("query all datasets.")
 		//搜索的属性要指定{"timestamp":{"unmapped_type":"date"}}
-		res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From((Page - 1) * PageSize).Size(PageSize).Do(ctx.Req.Context())
+		res, err := client.Search(TableName).SortBy(getSort(SortBy, ascending)).From(from).Size(Size).Do(ctx.Req.Context())
 		if err == nil {
 			searchJson, _ := json.Marshal(res)
 			log.Info("searchJson=" + string(searchJson))
-			result := makeDatasetResult(res, "", OnlyReturnNum)
-			ctx.JSON(200, result)
+			esresult := makeDatasetResult(res, "", OnlyReturnNum)
+			resultObj.Total = resultObj.PrivateTotal + esresult.Total
+			log.Info("query dataset es count=" + fmt.Sprint(esresult.Total) + " total=" + fmt.Sprint(resultObj.Total))
+			resultObj.Result = append(resultObj.Result, esresult.Result...)
+			ctx.JSON(200, resultObj)
 		} else {
 			log.Info("query es error," + err.Error())
 			ctx.JSON(200, "")
@@ -806,6 +857,42 @@ func searchDataSet(ctx *context.Context, TableName string, Key string, Page int,
 
 }
 
+func makePrivateDataSet(datasets []*models.Dataset, res *SearchRes, Key string) {
+	for _, dataset := range datasets {
+		record := make(map[string]interface{})
+
+		record["id"] = dataset.ID
+		userId := dataset.UserID
+
+		user, errUser := models.GetUserByID(userId)
+		if errUser == nil {
+			record["owerName"] = user.GetDisplayName()
+			record["avatar"] = user.RelAvatarLink()
+		}
+
+		repo, errRepo := models.GetRepositoryByID(dataset.RepoID)
+		if errRepo == nil {
+			log.Info("repo_url=" + repo.FullName())
+			record["repoUrl"] = repo.FullName()
+			record["avatar"] = repo.RelAvatarLink()
+		} else {
+			log.Info("repo err=" + errRepo.Error())
+		}
+
+		record["title"] = makeHighLight(Key, dataset.Title)
+		record["description"] = truncLongText(makeHighLight(Key, dataset.Description), true)
+
+		record["category"] = dataset.Category
+		record["task"] = dataset.Task
+		record["download_times"] = dataset.DownloadTimes
+		record["created_unix"] = dataset.CreatedUnix
+		record["updated_unix"] = repo.UpdatedUnix
+		record["updated_html"] = timeutil.TimeSinceUnix(repo.UpdatedUnix, "zh-CN")
+
+		res.Result = append(res.Result, record)
+	}
+}
+
 func makeDatasetResult(sRes *elastic.SearchResult, Key string, OnlyReturnNum bool) *SearchRes {
 	total := sRes.Hits.TotalHits.Value
 	result := make([]map[string]interface{}, 0)