From 2ff15d543fea3b31252e299dee4db2ca1d940593 Mon Sep 17 00:00:00 2001 From: wangyi15124 Date: Tue, 1 Nov 2022 19:14:39 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=B3=E6=97=B6=E6=90=9C=E7=B4=A2=E8=84=9A?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/static/xml/1.txt | 88 +++++++++++++++++ src/main/resources/static/xml/10.txt | 88 +++++++++++++++++ src/main/resources/static/xml/11.txt | 88 +++++++++++++++++ src/main/resources/static/xml/12.txt | 88 +++++++++++++++++ src/main/resources/static/xml/13.txt | 88 +++++++++++++++++ src/main/resources/static/xml/14.txt | 88 +++++++++++++++++ src/main/resources/static/xml/15.txt | 88 +++++++++++++++++ src/main/resources/static/xml/16.txt | 88 +++++++++++++++++ src/main/resources/static/xml/17.txt | 88 +++++++++++++++++ src/main/resources/static/xml/18.txt | 88 +++++++++++++++++ src/main/resources/static/xml/19.txt | 88 +++++++++++++++++ src/main/resources/static/xml/2.txt | 136 +++++++++++++++++++++++++++ src/main/resources/static/xml/20.txt | 88 +++++++++++++++++ src/main/resources/static/xml/3.txt | 88 +++++++++++++++++ src/main/resources/static/xml/4.txt | 118 +++++++++++++++++++++++ src/main/resources/static/xml/5.txt | 88 +++++++++++++++++ src/main/resources/static/xml/6.txt | 94 ++++++++++++++++++ src/main/resources/static/xml/7.txt | 88 +++++++++++++++++ src/main/resources/static/xml/8.txt | 88 +++++++++++++++++ src/main/resources/static/xml/9.txt | 88 +++++++++++++++++ 20 files changed, 1844 insertions(+) create mode 100644 src/main/resources/static/xml/1.txt create mode 100644 src/main/resources/static/xml/10.txt create mode 100644 src/main/resources/static/xml/11.txt create mode 100644 src/main/resources/static/xml/12.txt create mode 100644 src/main/resources/static/xml/13.txt create mode 100644 src/main/resources/static/xml/14.txt create mode 100644 src/main/resources/static/xml/15.txt create mode 100644 src/main/resources/static/xml/16.txt create mode 100644 src/main/resources/static/xml/17.txt create mode 100644 src/main/resources/static/xml/18.txt create mode 100644 src/main/resources/static/xml/19.txt create mode 100644 src/main/resources/static/xml/2.txt create mode 100644 src/main/resources/static/xml/20.txt create mode 100644 src/main/resources/static/xml/3.txt create mode 100644 src/main/resources/static/xml/4.txt create mode 100644 src/main/resources/static/xml/5.txt create mode 100644 src/main/resources/static/xml/6.txt create mode 100644 src/main/resources/static/xml/7.txt create mode 100644 src/main/resources/static/xml/8.txt create mode 100644 src/main/resources/static/xml/9.txt diff --git a/src/main/resources/static/xml/1.txt b/src/main/resources/static/xml/1.txt new file mode 100644 index 0000000..ddb0e6e --- /dev/null +++ b/src/main/resources/static/xml/1.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["dataJson"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${ resp.html.json().data.data }"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["page","keyword"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${pageNoData}","北京"],"shape":"variable"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${'https://www.chinaso.com/v5/general/v1/web/search?q=site:mp.weixin.qq.com+'+keyword+'&pn='+page+'&ps=15&bid=4151016307959391'}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${ dataJson.size() }","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","source","publish_time","abstract","author"],"variable-description":["","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${strUtils.contains(dataJson[i].title,keyword)?(strUtils.replaceAll(dataJson[i].title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(dataJson[i].title)}","${ dataJson[i].url }","${ dataJson[i].source }","${dateUtils.stampToDate(dataJson[i].timestamp+'000')}","${strUtils.contains( dataJson[i].snippet ,keyword)?(strUtils.replaceAll( dataJson[i].snippet ,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(dataJson[i].snippet)}","${'-'}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${title}","${url}","${\"微信公众号\"}","${publish_time}","${abstract}","${author}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/10.txt b/src/main/resources/static/xml/10.txt new file mode 100644 index 0000000..8f15ac6 --- /dev/null +++ b/src/main/resources/static/xml/10.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"500","timeout":"","response-charset":"","retryCount":"","retryInterval":"","header-name":["referer"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://cn.bing.com/search?q=\"+keyword+\"&filters=ex1%3a%22ez1%22&qs=SC&pq=nan&sc=10-3&cvid=005140B2204F4959940C48CB10B385E0&sp=1&first=\"+(page * 10+1)+\"&FORM=PERE\"}","proxy":"","header-value":["https://cn.bing.com/search?q=%e5%8d%97%e4%ba%ac&filters=ex1%3a%22ez1%22&qs=SC&pq=nan&sc=10-3&cvid=005140B2204F4959940C48CB10B385E0&sp=1&first=21&FORM=PERE1"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["datalist"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors(\"#b_results > li\")}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${''}","${''}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${''}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","abstract"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).selector('.b_title > h2 > a').text()}","${datalist.get(i).selector('.b_title > h2 > a').attr('href')}","${datalist.get(i).selector('.b_caption p').text()}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/11.txt b/src/main/resources/static/xml/11.txt new file mode 100644 index 0000000..576ff67 --- /dev/null +++ b/src/main/resources/static/xml/11.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"开始抓取1","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"utf-8","retryCount":"","retryInterval":"","cookie-name":["Cookie"],"cookie-description":[""],"header-name":["User-Agent"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${'https://www.baidu.com/s?wd=site%3Axiaohongshu.com%20'+keyword+'&pn='+startIndex+'0&oq=site%3Axiaohongshu.com%20'+keyword+'&ie=utf-8&rsv_pq=e2f0a475000042c5&rsv_t=9ea9dVLpUJnYTMsMoLr%2FJEKf5u9kVwWw0ClcmcocojFr%2FDLXSOQyIydSLKU'}","proxy":"","cookie-value":["BIDUPSID=FFCCEFFD6AD71B7D82B27246697129D3; PSTM=1661740480; BAIDUID=6504A43EB0274812F809031B8671878C:FG=1; BD_UPN=12314753; BDUSS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; BDUSS_BFESS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; H_WISE_SIDS=110085_209568_211986_212870_215730_216941_219942_219946_221410_221796_222298_222396_223064_223683_224055_224065_224086_225202_225640_225854_226026_226087_226589_226674_226722_227117_227592_227614_227864_227932_228374_228535_228650_228668_228978_229060_229155_229286_229363_229526_229691_229867_230020; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BAIDUID_BFESS=6504A43EB0274812F809031B8671878C:FG=1; B64_BOT=1; BDRCVFR[C5g0hgaJYCf]=bvBYpeu9K4bTAR3Qh4-I0; delPer=0; PSINO=3; BA_HECTOR=al8h2h852ka584a4agak0ggo1hj2sn519; ZFY=ieqEe:Ai8741EeaFdWPdP8JYsVLPfdZb2pa5AnSzsKW8:C; channel=bd.pe8.net; COOKIE_SESSION=2832_0_3_3_8_8_0_0_3_3_0_1_2833_0_54_0_1664186476_0_1664186530%7C9%23114260_19_1663339171%7C6; baikeVisitId=9b4f3a04-d583-4052-9723-8095c345a108; BD_CK_SAM=1; sugstore=1; H_PS_645EC=f709ExtDBjjkFE38TE3k8%2B41MnOFC0dMOW6Pc5%2BTWxSUTxI8HeJCOlKpHClk30Xoq46OZjUQVWvG; BDRCVFR[C0p6oIjvx-c]=sycO_VPkq2bfjf4njRdn1DLg1nkgv99; H_PS_PSSID=37378_36548_36460_37354_37495_36885_37486_37396_36786_37447_37371; BDSVRTM=39"],"header-value":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["datalist"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors('#content_left > div')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page","startIndex"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}","${page=='0'?'':(page-1)}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","source","publish_time","abstract"],"variable-description":["","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).selector('.c-container > div > h3 > a').text()}","${datalist.get(i).selector('.c-container > div > h3 > a').attr('href')}","${\"小红书\"}","${datalist.get(i).selector('.c-color-gray2').text()}","${datalist.get(i).selector('.content-right_8Zs40').text()}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${source}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/12.txt b/src/main/resources/static/xml/12.txt new file mode 100644 index 0000000..9a018db --- /dev/null +++ b/src/main/resources/static/xml/12.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","source_url","publish_time","descr","descr_temp","descr_data","abstract"],"variable-description":["","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${dataList.get(k).title}","${\"https://www.cls.cn/detail/\" + dataList.get(k).id}","${dateUtils.stampToDate(dataList.get(k).time +'000')}","${dataList.get(k).descr}","${strUtils.replaceAll(descr,'<em>','')}","${strUtils.replaceAll(descr_temp,'</em>','')}","${descr_data.length() <= 200 ? descr_data : descr_data.substring(0,200)}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${\"财联社\"}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"POST","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"raw","body-content-type":"application/json","loopCount":"","url":"${\"https://www.cls.cn/api/sw?app=CailianpressWeb&os=web&sv=7.7.5&sign=bf0f367462d8cd70917ba5eab3853bce\"}","proxy":"","request-body":"${'{\"type\":\"depth\",\"keyword\":\"'+keyword+'\",\"page\":'+(page-1)+',\"rn\":10,\"os\":\"web\",\"sv\":\"7.7.5\",\"app\":\"CailianpressWeb\"}'}","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","dataList"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.data.depth.data}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${dataList.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/13.txt b/src/main/resources/static/xml/13.txt new file mode 100644 index 0000000..8ee5cd3 --- /dev/null +++ b/src/main/resources/static/xml/13.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["source_url","publish_time","title","art_content"],"variable-description":["","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(k).Art_Url}","${datalist.get(k).Art_CreateTime}","${datalist.get(k).Art_Title}","${datalist.get(k).Art_Content}"],"shape":"variable"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","header-name":["Referer"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://searchapi.eastmoney.com/bussiness/Web/GetCMSSearchList?cb=jQuery35109950760626122721_1661172543372&keyword=\"+keyword+\"&type=8193&pageindex=\"+(page)+\"&pagesize=10&name=web&_=1661172543407\"}","proxy":"","header-value":["https://so.eastmoney.com/"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.substring(41,resp.html.length()-1).json()}","${json.Data}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${'东方财富'}","${publish_time}","${strUtils.contains(art_content,keyword)?(strUtils.replaceAll(art_content,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(art_content)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/14.txt b/src/main/resources/static/xml/14.txt new file mode 100644 index 0000000..8f087cc --- /dev/null +++ b/src/main/resources/static/xml/14.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","publish_time","source_url","source","contentDisp","content"],"variable-description":["","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(k).title.substring(1,datalist.get(k).title.length()-1)}","${dateUtils.stampToDate(datalist.get(k).createTime +'000')}","${\"https://www.gelonghui.com/live/\"+datalist.get(k).id}","${datalist.get(k).stocks.get(0).stockName}","${strUtils.replaceAll(datalist.get(k).contentDisp,'<hl>','')}","${strUtils.replaceAll(contentDisp,'</hl>','')}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${source}","${publish_time}","${strUtils.contains(content,keyword)?(strUtils.replaceAll(content,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(content)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"300","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://www.gelonghui.com/api/live/search/v2?keyword=\"+keyword+\"&page=\"+(page)+\"&count=10\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.result}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/15.txt b/src/main/resources/static/xml/15.txt new file mode 100644 index 0000000..c01ff26 --- /dev/null +++ b/src/main/resources/static/xml/15.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","source_url","publish_time","summary","abstract1","abstract","source"],"variable-description":["","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${dataList.get(k).title.replaceAll(\"</?[^>]+>\", \"\")}","${dataList.get(k).link}","${dateUtils.stampToDate(dataList.get(k).createTimestamp +'000')}","${dataList.get(k).summary}","${summary.length() <= 200 ? summary : summary.substring(0,200)}","${abstract1.replaceAll(\"</?[^>]+>\", \"\")}","${dataList.get(k).source}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${source}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"300","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://www.gelonghui.com/api/post/search/v4?keyword=\"+keyword+\"&isVipArticle=false&count=10&page=\"+(page)+\"&searchArea=title\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","dataList"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.result}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${dataList.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/16.txt b/src/main/resources/static/xml/16.txt new file mode 100644 index 0000000..03a5d0b --- /dev/null +++ b/src/main/resources/static/xml/16.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","1"],"shape":"variable"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"POST","sleep":"300","timeout":"","response-charset":"","retryCount":"","retryInterval":"","header-name":["cookie"],"header-description":[""],"body-type":"raw","body-content-type":"application/json","loopCount":"","url":"${\"https://www.cls.cn/api/sw?app=CailianpressWeb&os=web&sv=7.7.5&sign=bf0f367462d8cd70917ba5eab3853bce\"}","proxy":"","header-value":["${cookie}"],"request-body":"${'{\"type\":\"telegram\",\"keyword\":\"'+keyword+'\",\"page\":'+page+',\"rn\":10,\"os\":\"web\",\"sv\":\"7.7.5\",\"app\":\"CailianpressWeb\"}'}","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["source_url","publish_time","title1","title2","title","abstract1","abstract2","abstract"],"variable-description":["","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${\"https://www.cls.cn/detail/\" + dataList.get(k).id}","${dateUtils.stampToDate( dataList.get(k).time +'000')}","${dataList.get(k).descr}","${title1.substring(1,title1.indexOf('】'))}","${title2.replaceAll(\"</?[^>]+>\", \"\")}","${dataList.get(k).descr}","${abstract1.length() <= 200 ? abstract1 : abstract1.substring(0,200)}","${abstract2.replaceAll(\"</?[^>]+>\", \"\")}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json1","dataList"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json1.data.telegram.data}"],"shape":"variable"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${dataList.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${'格隆汇'}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${\"-\"}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/17.txt b/src/main/resources/static/xml/17.txt new file mode 100644 index 0000000..9bc76e7 --- /dev/null +++ b/src/main/resources/static/xml/17.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://cn.bing.com/videos/asyncv2?q=\"+keyword+\"&async=content&first=\"+page+\"&count=35&dgst=RowIndex_u6*ColumnIndex_u4*TotalWidth_u1192*OrdinalPosition_u34*ThumbnailWidth_u270*HeroContainerWidth_u1463*HeroContainerHeight_u275*HeroOnPage_b0*SlidesGridOnPage_b0*arn_u0*ayo_u0*cry_u1686*&IID=video.1&SFX=2&IG=B334CEAB86F542A2BA6167EE320E125D&CW=1519&CH=348&dgcw=1491&form=QBVR\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["datalist"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors('.dg_u')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["vediourl","videoorientationurl","videojson","imglist","imgjson","","","","","title","publish_time1","publish_time","source"],"variable-description":["","","","","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).selector('a').attr('href')}","${datalist.get(i).selector('a img').attr('src')}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",vediourl)}","${videojson.put(\"videoorientationurl\",videoorientationurl)}","${datalist.get(i).selector('a img').attr('alt')}","${datalist.get(i).selector('div.mc_vtvc_meta_block_area > div > div:nth-child(1) > span').text()}","${publish_time1.contains('次观看') || publish_time1.contains('浏览') ? '' : publish_time1}","${datalist.get(i).selector('div.mc_vtvc_meta_block_area > div > div:nth-child(2) > span').text()}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["videojson","title","publish_time","source"],"loopCount":"","output-value":["${videojson}","${title}","${publish_time}","${source}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/18.txt b/src/main/resources/static/xml/18.txt new file mode 100644 index 0000000..fdf71d5 --- /dev/null +++ b/src/main/resources/static/xml/18.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://image.baidu.com/search/acjson?tn=resultjson_com&logid=10483604795040733756&ipn=rj&ct=201326592&is=&fp=result&fr=&word=\"+keyword+\"&queryWord=\"+keyword+\"&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=&z=&ic=&hd=&latest=&copyright=&s=&se=&tab=&width=&height=&face=&istype=&qc=&nc=1&expermode=&nojc=&isAsync=&pn=90&rn=30&gsm=5a0000000000005a&1666579181622=\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.data}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","publish_time","videojson","imglist","imgjson","","","","",""],"variable-description":["","","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).fromPageTitle}","${datalist.get(i).thumbURL}","${datalist.get(i).bdImgnewsDate}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imgjson.put(\"imgurl\",url)}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",'')}","${videojson.put(\"videoorientationurl\",'')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","videojson","publish_time"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${videojson}","${publish_time}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"blue","condition":"${title != null && url != null}","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/19.txt b/src/main/resources/static/xml/19.txt new file mode 100644 index 0000000..7bc3784 --- /dev/null +++ b/src/main/resources/static/xml/19.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","header-name":["Cookie","Referer","User-Agent"],"header-description":["","",""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://so.toutiao.com/search?dvpf=pc&source=search_subtab_switch&keyword=\"+keyword+\"&pd=atlas&action_type=search_subtab_switch&page_num=\"+(page-1)+\"&search_id=2022102411264801015013416052D26757&from=gallery&cur_tab_title=gallery&rawJSON=1\"}","proxy":"","header-value":["_tea_utm_cache_4916=undefined; _S_DPR=1.25; _S_IPAD=0; MONITOR_WEB_ID=7156843505628530190; passport_csrf_token=8506d4b37fcefe68598d341fa50b3476; passport_csrf_token_default=8506d4b37fcefe68598d341fa50b3476; tt_webid=7156843505628530190; ttwid=1%7CcKwTgA4nrKSNy2DiS9Cn7Xpik-BnaGBfw34wTz5Itx8%7C1666581921%7Cb19d3198d43fec1a13002bf16b78b3d0e3c38b49b05029161f86eaba8ee986e7; _S_WIN_WH=1536_360","https://so.toutiao.com/search?dvpf=pc&source=search_subtab_switch&keyword=%E4%BB%8A%E5%A4%A9%E5%90%83%E4%BB%80%E4%B9%88&pd=atlas&action_type=search_subtab_switch&page_num=0&search_id=&from=gallery&cur_tab_title=gallery","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.rawData.data}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","videojson","imglist","imgjson","","","","",""],"variable-description":["","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).text}","${datalist.get(i).img_url}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imgjson.put(\"imgurl\",url)}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",'')}","${videojson.put(\"videoorientationurl\",'')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","publish_time","videojson"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${''}","${videojson}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/2.txt b/src/main/resources/static/xml/2.txt new file mode 100644 index 0000000..af17dc2 --- /dev/null +++ b/src/main/resources/static/xml/2.txt @@ -0,0 +1,136 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"根据关键词搜索","loopVariableName":"","method":"GET","sleep":"1000","timeout":"","response-charset":"","retryCount":"1","retryInterval":"2000","cookie-name":["cookie"],"cookie-description":[""],"header-name":["user-agent"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"https://m.weibo.cn/api/container/getIndex?containerid=231522type%3D1%26q%3D%23${keyword}%23&page_type=searchall&page=${pageNo}","proxy":"","cookie-value":["SUB=_2AkMUDYtbf8NxqwJRmfwTz2LhZIhyyw7EieKiUXqAJRMxHRl-yT9jqhYstRB6P42ltJExNYB2psyLl8ONit3Dytbl__Lo; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WWcnADUlTd4zwRxZivlmOy.; _s_tentry=passport.weibo.com; Apache=5401105109200.461.1666253868866; SINAGLOBAL=5401105109200.461.1666253868866; ULV=1666253868908:1:1:1:5401105109200.461.1666253868866:"],"header-value":["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["jsonhtml","data","parseArray"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["${jsonUtils.parseObject(resp.html)}","${jsonhtml.getJSONObject(\"data\")}","${data.getJSONArray(\"cards\")}"],"shape":"variable"} + + + + + + {"value":"自定义的关键词","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","pageNo"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["南京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"z","loopCount":"${parseArray}","loopStart":"0","loopEnd":"-1","awaitSleep":"100","shape":"loop"} + + + + + + {"value":"","exception-flow":"2","lineWidth":"2","line-style":"sharp","lineColor":"#00ff00","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["jsonobject"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${parseArray.getJSONObject(z)}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"含有card_group","loopVariableName":"","ziloopVariableName":"","variable-name":["cardgrouparray"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${jsonobject.getJSONArray(\"card_group\")}"],"shape":"variable"} + + + + + + {"value":"判断关键字,确定json类型","exception-flow":"2","lineWidth":"2","line-style":"sharp","lineColor":"#00ff00","condition":"${jsonobject.containsKey(\"card_group\") == true}","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"index","loopCount":"${cardgrouparray.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"300","shape":"loop"} + + + + + + {"value":"","exception-flow":"2","lineWidth":"2","line-style":"sharp","lineColor":"#00ff00","condition":"","transmit-variable":"1"} + + + + + + {"value":"获取内容详情页","loopVariableName":"","ziloopVariableName":"","variable-name":["source_url","url","mblog","abstract","publish_time","source","title"],"variable-description":["","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${cardgrouparray.getJSONObject(index).getString(\"scheme\")}","${source_url.split(\"\\\\?\")[0]}","${cardgrouparray.getJSONObject(index).getJSONObject(\"mblog\")}","${mblog.text}","${mblog.created_at}","${'微博'}","${mblog.user.screen_name}"],"shape":"variable"} + + + + + + {"value":"带有详情页url的","exception-flow":"2","lineWidth":"2","line-style":"sharp","lineColor":"#00ff00","condition":"${cardgrouparray.getJSONObject(index).containsKey(\"mblog\") == true && cardgrouparray.getJSONObject(index).containsKey(\"scheme\") == true}","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["source_url","source_url","mblog"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["${jsonobject.getString(\"scheme\")}","${source_url.split(\"\\\\?\")[0]}","${jsonobject.getJSONObject(\"mblog\")}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"2","lineWidth":"2","line-style":"sharp","lineColor":"#00ff00","condition":"${jsonobject.containsKey(\"card_group\") == false}","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","abstract","publish_time","source","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${publish_time}","${source}","${title}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/20.txt b/src/main/resources/static/xml/20.txt new file mode 100644 index 0000000..20cc95b --- /dev/null +++ b/src/main/resources/static/xml/20.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page","total_page"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}","${page== 1 ? 0 :((page-1)*10)}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","cookie-name":["cookie"],"cookie-description":[""],"header-name":["referer","user-agent"],"header-description":["",""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://www.douyin.com/aweme/v1/web/general/search/single/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_general&sort_type=0&publish_time=0&keyword=\"+keyword+\"&search_source=normal_search&query_correct_type=1&is_filter_search=0&from_group_id=6987870724073557255&offset=\"+total_page+\"&count=10\"}","proxy":"","cookie-value":["ttwid=1%7CP0JhRRY4Bs8OVUvN5L2Miz7mJjuyOsBXql6ASbJCbh4%7C1666344544%7Cb84871b088be071be4b63a80dcdb75a7397a8cb0128399106dd2c84a2de9560b; douyin.com; strategyABtestKey=1666344479.962; s_v_web_id=verify_l9iah9u8_X69Syl98_IJxX_4CI1_AVrP_CgPLlmPB5lK0; passport_csrf_token=34c8e93a61ceeb9398a14e8b3e86ec3e; passport_csrf_token_default=34c8e93a61ceeb9398a14e8b3e86ec3e; ttcid=f1880482f8d14a2dbd289c981a45db6a40; AB_LOGIN_GUIDE_TIMESTAMP=%221666344479785%22; _tea_utm_cache_2018=undefined; SEARCH_RESULT_LIST_TYPE=%22single%22; __ac_signature=_02B4Z6wo00f01OB8TFAAAIDBazfXsxb7GdjgXEjAAFtT0QprmLESRwztu7QYMLFpMSbB4Gybg4kQDPDgTnlNKwF9vN0Ogg.ZlIH9XC.D4xHb65cqQlAvz6Hvt1vFWUmPIiX3Nc9QBBtf.6Tt6d; tt_scid=i3Pr6JT5tgEUsW7RRrlo3QPivVfT.XPI13V5JTns4LeMinDdpC2kPzCw22SZLiAs9930; msToken=yh8_iPx7x2jKq-VUhgHlsQKJlCn8OVxzsHNHM16tb5SCo0pRv80zK3BeBtVDS87eObvOwhS1gbTkLZehfoQ18uot1MBXNo9PQJoW6hvPrjPTGHN2_gfXabK4TRkUTg==; msToken=kieZ9kVyg6pPT3wUDqNwuIjnV1JlEqHmbQhKiWHOLIrdKx3gj-efmNJh6I14APcGbeQ-_CZp3hla-MKevOv0b37exw2k6g11FsDmJIsF4hftlQ-WXnCjN9u3jg0JCw==; __ac_nonce=0635408b200f4d9736d24; home_can_add_dy_2_desktop=%220%22"],"header-value":["https://www.douyin.com/search/%E8%8B%8F%E5%B7%9E?source=normal_search&aid=66cd7d9d-5ae0-46ec-9642-d4224c81d693&enter_from=recommend&focus_method=&gid=6987870724073557255","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.data}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","videojson","publish_time","author","url"],"loopCount":"","output-value":["${title}","${videojson}","${publish_time}","${author}","${'https://www.douyin.com/video/' + source_url}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","video_url","videoorientationurl1","videoorientationurl","publish_time","author","videojson","imglist","imgjson","","","","","","source_url"],"variable-description":["","","","","","","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).aweme_info.desc}","${datalist.get(i).aweme_info.video.play_addr.url_list.get(0)}","${datalist.get(i).aweme_info.video.cover.url_list}","${videoorientationurl1.get(0)}","${dateUtils.stampToDate(datalist.get(i).aweme_info.create_time +'000')}","${datalist.get(i).aweme_info.author.nickname}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imgjson.put(\"imgurl\",'')}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",video_url)}","${videojson.put(\"videoorientationurl\",videoorientationurl)}","${datalist.get(i).aweme_info.aweme_id}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"blue","condition":"${source_url != null}","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/3.txt b/src/main/resources/static/xml/3.txt new file mode 100644 index 0000000..d98b4f8 --- /dev/null +++ b/src/main/resources/static/xml/3.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"抓取列表","loopVariableName":"","method":"GET","sleep":"200","timeout":"3000","response-charset":"","retryCount":"3","retryInterval":"200","cookie-name":["Cookie"],"cookie-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"http://www.baidu.com/s?ie=utf-8&medium=1&rtt=1&bsst=1&rsv_dl=news_t_sk&cl=2&wd=\"+keyword+\"&tn=news&rsv_bp=1&tfflag=0&tngroupname=organic_news&newVideo=12&goods_entry_switch=1&pn=\"+startIndex+\"0\"}","proxy":"","cookie-value":["BIDUPSID=FFCCEFFD6AD71B7D82B27246697129D3; PSTM=1661740480; BAIDUID=6504A43EB0274812F809031B8671878C:FG=1; BD_UPN=12314753; BDUSS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; BDUSS_BFESS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; H_WISE_SIDS=110085_209568_211986_212870_215730_216941_219942_219946_221410_221796_222298_222396_223064_223683_224055_224065_224086_225202_225640_225854_226026_226087_226589_226674_226722_227117_227592_227614_227864_227932_228374_228535_228650_228668_228978_229060_229155_229286_229363_229526_229691_229867_230020; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BAIDUID_BFESS=6504A43EB0274812F809031B8671878C:FG=1; B64_BOT=1; BDRCVFR[C5g0hgaJYCf]=bvBYpeu9K4bTAR3Qh4-I0; delPer=0; PSINO=3; BA_HECTOR=al8h2h852ka584a4agak0ggo1hj2sn519; ZFY=ieqEe:Ai8741EeaFdWPdP8JYsVLPfdZb2pa5AnSzsKW8:C; channel=bd.pe8.net; COOKIE_SESSION=2832_0_3_3_8_8_0_0_3_3_0_1_2833_0_54_0_1664186476_0_1664186530%7C9%23114260_19_1663339171%7C6; baikeVisitId=9b4f3a04-d583-4052-9723-8095c345a108; BD_CK_SAM=1; sugstore=1; H_PS_645EC=f709ExtDBjjkFE38TE3k8%2B41MnOFC0dMOW6Pc5%2BTWxSUTxI8HeJCOlKpHClk30Xoq46OZjUQVWvG; BDRCVFR[C0p6oIjvx-c]=sycO_VPkq2bfjf4njRdn1DLg1nkgv99; H_PS_PSSID=37378_36548_36460_37354_37495_36885_37486_37396_36786_37447_37371; BDSVRTM=39"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["titles","urls","times","sources","totals","total","pages","pages","abstracts","imgurls"],"variable-description":["标题","摘要","原始链接","来源网站","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors(\"div>#content_left>div>div>.news-title_1YtI1\").texts()}","${resp.selectors(\"div>#content_left>div>div>.news-title_1YtI1>a\").attrs(\"href\")}","${resp.selectors(\"div>#content_left>div>div>div>div>span.c-gap-right-xsmall\").texts()}","${resp.selectors(\"div>#content_left>div>div>div>div>div>a>span\").texts()}","${resp.selector(\"#header_top_bar > span\").text()}","${strUtils.substring(totals,totals.indexOf(\"相关资讯\")+4,totals.lastIndexOf(\"个\"))}","${(strUtils.toInt(total)%10)==0?strUtils.toInt(total)/10:(strUtils.toInt(total)/10)+1}","${strUtils.toInt(pages+'')>2?2:pages}","${resp.selectors(\"div>#content_left>div>div>div>div>span.c-color-text\").texts()}","${resp.selectors(\"#content_left>div\")}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"j","loopCount":"${titles.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","source","time","time","time","publish_time","abstract","imgurl","videojson","imglist","imgjson","","","","",""],"variable-description":["","","","","","","","","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${titles[j]}","${urls[j]}","${sources[j]}","${times[j]}","${strUtils.contains(time,\"昨天\")?(strUtils.substring(dateUtils.minusDay(1),0,10)+\" \"+strUtils.substring(time,2,time.length())+\":00\"):time}","${strUtils.contains(time,\"前天\")?(strUtils.substring(dateUtils.minusDay(2),0,10)+\" \"+strUtils.substring(time,2,time.length())+\":00\"):time}","${dateUtils.FormatDate(time)}","${abstracts[j]}","${imgurls.get(j+1).selector('div>div .c-span3 img').attr('src')}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imgjson.put(\"imgurl\",imgurl)}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",'')}","${videojson.put(\"videoorientationurl\",'')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page","startIndex"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}","${page=='0'?'':(page-1)}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author","videojson"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${source}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${'-'}","${videojson}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/4.txt b/src/main/resources/static/xml/4.txt new file mode 100644 index 0000000..68506e5 --- /dev/null +++ b/src/main/resources/static/xml/4.txt @@ -0,0 +1,118 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"抓取列表","loopVariableName":"","method":"GET","sleep":"200","timeout":"3000","response-charset":"","retryCount":"3","retryInterval":"200","cookie-name":["Cookie"],"cookie-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${'https://so.toutiao.com/search?keyword='+keyword+'&pd=information&source=search_subtab_switch&dvpf=pc&aid=4916&page_num='+(page-1)}","proxy":"","cookie-value":["MONITOR_WEB_ID=7136060905808446989; _S_DPR=2; _S_IPAD=0; _S_WIN_WH=1372_739; ttwid=1%7CDU8QP2eAdg2wjFnB_ggNlqGibkM_DPTXGw1bvFbqUt8%7C1661493660%7C3e9e6f880b1d240b10612e18d44b39f2787f082f5438e6719efbb984559f96f4"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["page","keyword"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${pageNoData}","北京"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["orignalhtml","datalist"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html}","${orignalhtml.selectors('script[data-for=s-result-json]')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"j","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["html","sub_html","data_json_object","title","source_url","datetime","media_name","abstract","imgurl","videojson","imglist","imgjson","","","",""],"variable-description":["","","","","","","","","","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist[j]+''}","${strUtils.substring(html,strUtils.indexOf(html,'\"hydrate\">')+10,strUtils.length(html)-9)}","${jsonUtils.parseObject(sub_html)}","${data_json_object.data.title}","${data_json_object.data.source_url}","${data_json_object.data.datetime}","${data_json_object.data.media_name}","${data_json_object.data.abstract}","${data_json_object.data.image_url}","${jsonUtils.createJsonObject()}","${jsonUtils.createJsonArray()}","${jsonUtils.createJsonObject()}","${imgjson.put(\"imgurl\",imgurl)}","${imglist.add(imgjson)}","${videojson.put(\"imglist\",imglist)}","${videojson.put(\"vediourl\",'')}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","source_url","datetime","datalist"],"loopCount":"","output-value":["${title}","${source_url}","${datetime}","${data_json_object}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","classify","website_id","websitelogo"],"variable-description":["","","",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.new_website_type}","${json.id}","${json.website_ico}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["",""],"loopCount":"","output-value":["${resp.html}","${url}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"修改关键词状态","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"http://192.168.71.61:6135/spider_factory/baidu/keywordStatus?keyword=\"+keyword}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"http://192.168.71.61:6135/spider_factory/baidu/keywordStatus?keyword=\"+rs[i].keyword}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","publish_time","source","abstract","author","videojson"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${datetime}","${'今日头条'}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${media_name}","${videojson}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"blue","condition":"${abstract!=null||title!=null}","transmit-variable":"1"} + + + + diff --git a/src/main/resources/static/xml/5.txt b/src/main/resources/static/xml/5.txt new file mode 100644 index 0000000..8910d77 --- /dev/null +++ b/src/main/resources/static/xml/5.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"抓取列表","loopVariableName":"","method":"GET","sleep":"200","timeout":"3000","response-charset":"","retryCount":"3","retryInterval":"200","cookie-name":["Cookie"],"cookie-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"http://www.baidu.com/s?ie=utf-8&medium=2&rtt=1&bsst=1&rsv_dl=news_t_sk&cl=2&wd=\"+keyword+\"&tn=news&rsv_bp=1&tfflag=0&tngroupname=organic_news&newVideo=12&goods_entry_switch=1&pn=\"+startIndex+\"0\"}","proxy":"","cookie-value":["BIDUPSID=FFCCEFFD6AD71B7D82B27246697129D3; PSTM=1661740480; BAIDUID=6504A43EB0274812F809031B8671878C:FG=1; BD_UPN=12314753; BDUSS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; BDUSS_BFESS=NneHRaYzJNVnVCQ1lOa2RLNkozYlBNQkJMeHVqMzFOZm96SH5mQWlKajZia2RqSVFBQUFBJCQAAAAAAAAAAAEAAAB~B2D2tcjIscqntcSwrgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPrhH2P64R9jb0; H_WISE_SIDS=110085_209568_211986_212870_215730_216941_219942_219946_221410_221796_222298_222396_223064_223683_224055_224065_224086_225202_225640_225854_226026_226087_226589_226674_226722_227117_227592_227614_227864_227932_228374_228535_228650_228668_228978_229060_229155_229286_229363_229526_229691_229867_230020; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BAIDUID_BFESS=6504A43EB0274812F809031B8671878C:FG=1; B64_BOT=1; BDRCVFR[C5g0hgaJYCf]=bvBYpeu9K4bTAR3Qh4-I0; delPer=0; PSINO=3; BA_HECTOR=al8h2h852ka584a4agak0ggo1hj2sn519; ZFY=ieqEe:Ai8741EeaFdWPdP8JYsVLPfdZb2pa5AnSzsKW8:C; channel=bd.pe8.net; COOKIE_SESSION=2832_0_3_3_8_8_0_0_3_3_0_1_2833_0_54_0_1664186476_0_1664186530%7C9%23114260_19_1663339171%7C6; baikeVisitId=9b4f3a04-d583-4052-9723-8095c345a108; BD_CK_SAM=1; sugstore=1; H_PS_645EC=f709ExtDBjjkFE38TE3k8%2B41MnOFC0dMOW6Pc5%2BTWxSUTxI8HeJCOlKpHClk30Xoq46OZjUQVWvG; BDRCVFR[C0p6oIjvx-c]=sycO_VPkq2bfjf4njRdn1DLg1nkgv99; H_PS_PSSID=37378_36548_36460_37354_37495_36885_37486_37396_36786_37447_37371; BDSVRTM=39"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["titles","urls","times","sources","totals","total","pages","pages","abstracts"],"variable-description":["标题","摘要","原始链接","来源网站","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors(\"div>#content_left>div>div>.news-title_1YtI1\").texts()}","${resp.selectors(\"div>#content_left>div>div>.news-title_1YtI1>a\").attrs(\"href\")}","${resp.selectors(\"div>#content_left>div>div>div>div>span.c-gap-right-xsmall\").texts()}","${resp.selectors(\"div>#content_left>div>div>div>div>div>a>span\").texts()}","${resp.selector(\"#header_top_bar > span\").text()}","${strUtils.substring(totals,totals.indexOf(\"相关资讯\")+4,totals.lastIndexOf(\"个\"))}","${(strUtils.toInt(total)%10)==0?strUtils.toInt(total)/10:(strUtils.toInt(total)/10)+1}","${strUtils.toInt(pages+'')>2?2:pages}","${resp.selectors(\"div>#content_left>div>div>div>div>span.c-color-text\").texts()}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"j","loopCount":"${titles.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","source","time","time","time","publish_time","abstract"],"variable-description":["","","","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${titles[j]}","${urls[j]}","${sources[j]}","${times[j]}","${strUtils.contains(time,\"昨天\")?(strUtils.substring(dateUtils.minusDay(1),0,10)+\" \"+strUtils.substring(time,2,time.length())+\":00\"):time}","${strUtils.contains(time,\"前天\")?(strUtils.substring(dateUtils.minusDay(2),0,10)+\" \"+strUtils.substring(time,2,time.length())+\":00\"):time}","${dateUtils.FormatDate(time)}","${abstracts[j]}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page","startIndex"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}","${page=='0'?'':(page-1)}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${source}","${publish_time}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/6.txt b/src/main/resources/static/xml/6.txt new file mode 100644 index 0000000..901cb5f --- /dev/null +++ b/src/main/resources/static/xml/6.txt @@ -0,0 +1,94 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["total_page","keyword"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${pageNoData}","北京"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["list_data"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json().data.data}"],"shape":"variable"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${list_data.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","title","summary","url","source","publish_time","extend","source_flag","site_name","snippet"],"variable-description":["标题","","摘要","原始链接","来源网站","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${strUtils.replaceAll(list_data[i].title,'<em>','')}","${strUtils.replaceAll(title,'</em>','')}","${list_data[i].snippet}","${list_data[i].url}","${list_data[i].source}","${dateUtils.stampToDate(list_data[i].timestamp+'000')}","${list_data[i].extend}","${strUtils.indexOf(extend,'site_name')}","${source_flag!=-1?(list_data[i].extend.json().site_name):source}","${list_data.get(i).snippet}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"","timeout":"","response-charset":"","retryCount":"","retryInterval":"","cookie-name":["Cookie"],"cookie-description":[""],"header-name":["Host"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${'http://www.chinaso.com/v5/general/v1/web/search?q='+keyword+'&pn='+total_page+'&ps=15&force=0&stime=24h&etime=now'}","proxy":"","cookie-value":["uid=CgqASWJmH0KGd3qqDSdIAg==; cookie_name=122.96.102.43.1650859843374970; wdcid=7e72f38d9bc6dc56; wdlast=1652238633; Hm_lvt_91fa1aefc990a9fc21c08506e5983ddf=1652238633; __WWW_CARD__DISSTATUS__=false; __WWW_nav__DISSTATUS__=false; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221805eea194944-0d246d8dddcd0b-535b0612-1296000-1805eea194a99e%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%221805eea194944-0d246d8dddcd0b-535b0612-1296000-1805eea194a99e%22%7D"],"header-value":["www.chinaso.com"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["url","title","source","publish_time","abstract","author"],"loopCount":"","output-value":["${url}","${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source}","${publish_time}","${strUtils.contains(snippet,keyword)?(strUtils.replaceAll(snippet,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(snippet)}","${'-'}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["content"],"loopCount":"","output-value":["${content}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/7.txt b/src/main/resources/static/xml/7.txt new file mode 100644 index 0000000..a0a8b39 --- /dev/null +++ b/src/main/resources/static/xml/7.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["authorName","publish_time","source_url","title","source","summary"],"variable-description":["","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist1.get(k).fields.AuthorName}","${dateUtils.stampToDate(datalist1.get(k).fields.PublishTime +'000')}","${datalist1.get(k).fields.url}","${datalist1.get(k).fields.title.replaceAll(\"</?[^>]+>\", \"\")}","${datalist1.get(k).fields.webname}","${datalist1.get(k).fields.summary}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${source}","${publish_time}","${strUtils.contains(summary,keyword)?(strUtils.replaceAll(summary,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(summary)}","${authorName}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"POST","sleep":"300","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://vtm.sm.cn/blm/vsearch-api-390/news?uc_param_str=dnntnwvepffrbijbprsvchgputdemennosstodcaaagidsdieinipi&start=\"+(page*10)+\"&method=hot_news&hit=10&query=\"+keyword+\"&hid=c1ba2f9f9e104793ad584ef09017d5d2&uuid=2c0fb4e4c3aeb7b59dc3b2795657e38b%7C%7C%7C1661225116\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist1"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.data.list}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${datalist1.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/8.txt b/src/main/resources/static/xml/8.txt new file mode 100644 index 0000000..a0a8b39 --- /dev/null +++ b/src/main/resources/static/xml/8.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["authorName","publish_time","source_url","title","source","summary"],"variable-description":["","","","","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist1.get(k).fields.AuthorName}","${dateUtils.stampToDate(datalist1.get(k).fields.PublishTime +'000')}","${datalist1.get(k).fields.url}","${datalist1.get(k).fields.title.replaceAll(\"</?[^>]+>\", \"\")}","${datalist1.get(k).fields.webname}","${datalist1.get(k).fields.summary}"],"shape":"variable"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${source_url}","${source}","${publish_time}","${strUtils.contains(summary,keyword)?(strUtils.replaceAll(summary,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(summary)}","${authorName}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"POST","sleep":"300","timeout":"","response-charset":"","retryCount":"","retryInterval":"","body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://vtm.sm.cn/blm/vsearch-api-390/news?uc_param_str=dnntnwvepffrbijbprsvchgputdemennosstodcaaagidsdieinipi&start=\"+(page*10)+\"&method=hot_news&hit=10&query=\"+keyword+\"&hid=c1ba2f9f9e104793ad584ef09017d5d2&uuid=2c0fb4e4c3aeb7b59dc3b2795657e38b%7C%7C%7C1661225116\"}","proxy":"","request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["json","datalist1"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["${resp.html.json()}","${json.data.list}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"k","loopCount":"${datalist1.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file diff --git a/src/main/resources/static/xml/9.txt b/src/main/resources/static/xml/9.txt new file mode 100644 index 0000000..8f15ac6 --- /dev/null +++ b/src/main/resources/static/xml/9.txt @@ -0,0 +1,88 @@ + + + + + {"spiderName":"未定义名称","submit-strategy":"random","threadCount":"1"} + + + + + + + {"shape":"start"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["keyword","page"],"variable-description":["",""],"loopCount":"","ziloopCount":"","variable-value":["北京","${pageNoData}"],"shape":"variable"} + + + + + + {"value":"开始抓取","loopVariableName":"","method":"GET","sleep":"500","timeout":"","response-charset":"","retryCount":"","retryInterval":"","header-name":["referer"],"header-description":[""],"body-type":"none","body-content-type":"text/plain","loopCount":"","url":"${\"https://cn.bing.com/search?q=\"+keyword+\"&filters=ex1%3a%22ez1%22&qs=SC&pq=nan&sc=10-3&cvid=005140B2204F4959940C48CB10B385E0&sp=1&first=\"+(page * 10+1)+\"&FORM=PERE\"}","proxy":"","header-value":["https://cn.bing.com/search?q=%e5%8d%97%e4%ba%ac&filters=ex1%3a%22ez1%22&qs=SC&pq=nan&sc=10-3&cvid=005140B2204F4959940C48CB10B385E0&sp=1&first=21&FORM=PERE1"],"request-body":"","follow-redirect":"1","tls-validate":"1","cookie-auto-set":"1","repeat-enable":"0","shape":"request"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["datalist"],"variable-description":[""],"loopCount":"","ziloopCount":"","variable-value":["${resp.selectors(\"#b_results > li\")}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"输出","loopVariableName":"","tableName":"","csvName":"","csvEncoding":"GBK","output-name":["title","url","source","publish_time","abstract","author"],"loopCount":"","output-value":["${strUtils.contains(title,keyword)?(strUtils.replaceAll(title,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(title)}","${url}","${''}","${''}","${strUtils.contains(abstract,keyword)?(strUtils.replaceAll(abstract,keyword,'<b class=\"key\" style=\"color:red\">'+keyword+'</b>')):(abstract)}","${''}"],"output-all":"0","output-database":"0","output-csv":"0","shape":"output"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"循环","loopItem":"","loopVariableName":"i","loopCount":"${datalist.size()}","loopStart":"0","loopEnd":"-1","awaitSleep":"0","shape":"loop"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"定义变量","loopVariableName":"","ziloopVariableName":"","variable-name":["title","url","abstract"],"variable-description":["","",""],"loopCount":"","ziloopCount":"","variable-value":["${datalist.get(i).selector('.b_title > h2 > a').text()}","${datalist.get(i).selector('.b_title > h2 > a').attr('href')}","${datalist.get(i).selector('.b_caption p').text()}"],"shape":"variable"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + + + {"value":"","exception-flow":"0","lineWidth":"2","line-style":"sharp","lineColor":"black","condition":"","transmit-variable":"1"} + + + + \ No newline at end of file