From 230545f8a9e75e488bb85cc924b88860982ec193 Mon Sep 17 00:00:00 2001 From: ngoura Date: Thu, 15 Jun 2017 17:22:31 +0200 Subject: [PATCH] add reglage for crawl --- .eslintrc.json | 4 +- .meteor/packages | 1 + .meteor/versions | 2 + .~lock.Description.docx# | 1 + Description.docx | Bin 0 -> 13140 bytes imports/api/account/methods.js | 13 ++++- imports/api/config/methods.js | 34 ++++++++++++ imports/api/crawl/crawlWebsite.js | 3 +- imports/api/indexation/elasticSearchConfig.js | 16 +++++- imports/api/indexation/indexGeneric.js | 2 + imports/api/indexation/indexWebsite.js | 51 ++++++++++++++---- imports/api/testSearch/methods.js | 26 +++++++-- imports/collections/configCollection.js | 29 ++++++++++ imports/collections/testSearchCollection.js | 2 +- imports/startup/server/register-api.js | 1 + .../ui/components/testSearch/testSearch.html | 3 ++ .../ui/components/testSearch/testSearch.js | 5 ++ imports/ui/pages/admin/account/add/add.js | 1 + .../ui/pages/admin/indexation/api/api.html | 2 +- .../ui/pages/admin/indexation/main/main.html | 2 +- .../ui/pages/admin/indexation/site/site.html | 21 +++++++- .../ui/pages/admin/indexation/site/site.js | 41 ++++++++++++++ imports/ui/pages/login/login.js | 7 +++ package.json | 2 +- yarn.lock | 15 ++---- 25 files changed, 249 insertions(+), 35 deletions(-) create mode 100644 .~lock.Description.docx# create mode 100644 Description.docx create mode 100644 imports/api/config/methods.js create mode 100644 imports/collections/configCollection.js diff --git a/.eslintrc.json b/.eslintrc.json index 50e65be..ab25d3a 100755 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -32,6 +32,8 @@ "import/extensions": ["off", "never"], "import/no-extraneous-dependencies": "off", "no-underscore-dangle": "off", - "class-methods-use-this": "off" + "class-methods-use-this": "off", + "no-param-reassign": "off", + "meteor/no-session": "off" } } diff --git a/.meteor/packages b/.meteor/packages index 39ab7d0..8ec6f40 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -45,3 +45,4 @@ practicalmeteor:chai deanius:promise dynamic-import aldeed:tabular +ajduke:bootstrap-tagsinput diff --git a/.meteor/versions b/.meteor/versions index e4c551e..30e26b4 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -1,5 +1,6 @@ accounts-base@1.3.0 accounts-password@1.3.6 +ajduke:bootstrap-tagsinput@0.7.1 alanning:roles@1.2.16 aldeed:autoform@6.2.0 aldeed:collection2-core@2.0.1 @@ -99,6 +100,7 @@ templating-tools@1.1.2 tmeasday:check-npm-versions@0.3.1 tmeasday:test-reporter-helpers@0.2.1 tracker@1.1.3 +twbs:bootstrap@3.3.6 ui@1.0.13 underscore@1.0.10 url@1.1.0 diff --git a/.~lock.Description.docx# b/.~lock.Description.docx# new file mode 100644 index 0000000..077daab --- /dev/null +++ b/.~lock.Description.docx# @@ -0,0 +1 @@ +,ngoura,ngoura,15.06.2017 09:13,file:///home/ngoura/.config/libreoffice/4; \ No newline at end of file diff --git a/Description.docx b/Description.docx new file mode 100644 index 0000000000000000000000000000000000000000..002aba3f860b5b937071a49a0ea7fd043f8a557a GIT binary patch literal 13140 zcma)j1ymi$)-LYu?(XgyJh(dq65QQAIKkar1Hs+xAi>?;-67ZyGxz?Jnf2bhx6kTc z)m8QFuCsU7F6ploq(Q;ZfS{nDfS7bgKLY(Guy1L7dm~E+Muzuu`DE8eP-Y~Eb&o8= z6c3krRnadsZ6V`nl3u>iKTjdjWF!l<)>ai@nibT#2v{HX_DJ(r@SfY3@hX zmS$Zy8vk`hbn0QUA2ZUW94*bj0|PGYoSFYQ9d$)V`3r*P{OhG?(NDaC+thrs&84fZ z!Cd~Ru~f0}xU2;)UzDG)hrbITzEWa{h=T2`Y(p6TKDb zVr-&z7W~@0D=<Pag_kAz_o8Lg2hJX43|nxAzOlTT8{;-}*-ugc(~nDx~mF>GOv)4LZd&@gsE* zdfQZRf^pnV$Z1MIf50Cj%=-b0P@bEIFY};u#0lu2i?p6V@eK)12}Gn|G&sWxWR7s3 z0Ig_0Y$a2e2dfO#`PGC1H=AslP1k_tV*R>2ci+wod`QNivS)O*rIY_O>yf;@c_`25 z{bp=PC_xrz9#&u=AQ-TJGd}WvjqhOO=xAnb;_z}IPoh?i#{y4JR{97Me2lCMuRexe_V>T*(id?}3de)i0ugRP78 z!%|3o956goaUggD2Fzt^C;!h@w^ecR5-PH7MxuK}pr@_}S)L9{3o4;2j)C5a&&K-+ zjg=Z>%wRC`s{_*&#zCs!!f*<#KaH@N6sB$-kZWVW+s8Ti^+^+_tsNPgMfST($7xp) zrLsxKNDg)It!dc|f9Y;sQ&Z@_d*`Zce_8*{JDQ;X<{k9^ddJb!$jXTEk2CYThfXwQ z!*MyW+V9li99(ui`t>&9Z|CI}k12?c^0%p?cChf*ohTll^Fv+Xt|qaH(-1; z%k~S>bKjTOlr)_pR2+#AWUEh)>WB5~h43KbKP!_sqvfCcsnhD1YIuUB6eKx2T@G7V z(M3lh65l8Ltk$7kc9e80*aUU!3Zh&L?bv@Sq%6;sb&L%WF3SE1a{$aZ*GkkBrjcrL z9!trVq&0uOQg(_{$(KcUSO_M~UGU+Ea^v`y=~t5CGsaezs{28%VOkQIsNCG)NzrZu zfZ7Ydn66ia7oxEwJThoIA5p%k!l1L!)b!B!V&dC#c>g5S6@7iI?vMMQC%X_mRl4&N zu9a+#x??xm(yz`3KB#l>&E9kRzi5Go!g?jk;lIyW*Yd97(fSQZ2cF4|+kJ=8yF9a8 z*VcO}U(Y~aUFp1Y-=GOXDQ>++V}?E+FJEs%UtQ~jG%nkMK=%ax$TZZCYdUq6VavQo z{zdd0oxF9UsjV}!N1aL}Rky&azhR^fZJE555=hJ*SR$q01eFUUrZW&I4_IswI1H8f zf`W)tMkTF&fffHE&1Yq^Gz4lsK&YCbjHF&M;c7xdbmA-L&Dd9UU4K5G+vDS|sT#OQYelE5~KOwk)&@gE9x11J||W5a-Hczi*BIb%G75iR5<^uc{0YEaB>;Va4h zF#-euql2rq<(dZ5qyWq)f2IaSNbWck1LDdqMcXOz!8T1{C%IzRYfv%SxT@KKC&7*r zm`m*jzrv`AHzUL?N`;{ZOwDw)lVpb|2zy&=RlFaG=I1GqO6U=B7?r`7-a1}M3($Nx zDMI+h#eJBMVtU3j{v>M&^LVSXi(qBTF6(ya)UyzsG)YM6-LBX&+>!|q!eK_+!0Vx5 z+Y=Gu3BIfhEchZawWmldh1E@RCg{!Tfj^6%Az5Ck+2ttx0rwrLN*;3SZi2oIRgf?Q~k!! zMsqqmBP6NLyRsQxyP5GLa~!m5;1nk%v6CA429+e(Xj)9=Pu(h0DvjxK^(vyU^iC-byX$sTxE&1s8zdo@{OUL`6W@kB?>x)s2fo`&Cs^#>^C7}pwB+>Mjx=x zfD+>?b4&1i!npEr#7mNo>~U=CH;otk9$sQClF#(FBW}ogCZNPUrAJDs7C%xB&ANc? zfwQ3JN5yW0a&#)YqlO5%`6WQwhG6N7JA@RoHK{A=#FafLedlA>rdRZ(gib!As!mQi z-*lIfz>(;3^im0;`5MsJ5UzW7BW{t~WCRLbUpoI_b%{bNF*2lVuA67w?w;g&2QQqf zrmpNTJzal-&&!7-x$>aBX>2pH%XsusPk{>Z77^hlx8qXKh=h@oby*h?7o@+MSkM`HoxU%ne=5RQ*|a_BmNy zYxP-XV^&wq!%_1K`6pv@dR8HvMKg9m)k z#H|H<(M(xWhI892)U$K>RW zOB^U(Yx-SzMJtf%v%yBZMb`4Uy|@jLN8+Qm*d)Zz>|&-5$V-n$NkCGelR#*Y>8_l6 z*W^_~qvXE4F22B0=YXuNxtzNMKxp)Ys?B+i+1OSLBPcTc4|jcn3V8;1z1V81PdO3y zs%L?eZCO6QmZIcsVz0@!Lz9*y2GfelR%v88ZGJuX>R+%+dq?9E>;N!W^lI2%u^Aub zSr9m?uH9m{&yOkv`S>%g=VV0NV2qg{ge*1yEHUBNkopym_Eks%a|%rl>ng8x2(-R+ zYlS2h7=p%Waddzq-)pv0Y@YR`z0UmU0^9Bm;7jlD@Bs4|+qJNd4kKw)`k@Bvz1({%#Zr_wda5mK|xLT&1Iga9qqFO!OzL3 zPSvwobu{m$rl`OO2dy9ONl8W<5kLAQkiqy$GqB1uhJAfL;fR|8UB=u+ZHlYvQRK6zD^y}`f+q+z;I)cFeZy$N-KUAF>&KM{ZwEW$0d4>>~~;Mujs zm$MfiDGP}c7UH2;pNV?E8$>@?>57Krq8})Uf>w_(QteovLJF1cKJui2((5UXL1-;D z+4ci11++Y^{=7!V?5>o*|>$MA1X1 z4)Xc}ne`e_=x=jH#^p|IexjYbLk!P>4i+HeM+Qog@0m(XBFqwG-m3pFtW(XEE!-02 zb}=n66pCbG*#=!wn*iu5;(I~*8N{~Qp+Ps@jFeDJAlYd?93;i@8uz#&vXXHvsTSlt ztkm{WT-3axoWtLEI`#GKqzpDQ-SBAU`sKjlyRl(25(J?t) z#mVEJRM5pv`Oua` zl($K29d=;JAx}-Y1dfM3=yT4kIfI5_)CQ#{s)_YcsNF!hTP;dV_Xf>7o;x#O1IZnj7P?6LTMYV|(}M>W}kxKO^95fH9@kDqth8si9^Yc2 zFDNqESLfBvCgA5vead=JkHMNNj#_?Q72pTnd}(yre&Ua#Zj|M>d{vs06|f6KImNaU zKe5p{5ySQntViH<4wANC%IyrZ@#+7_DL?5PB6TKMWdIFJx*J)2KJ73o?zU z-P<7yi~ixC(2vEP%TmQi;D2Ji3c;ecB1WM7zix*FLay@ZluRYWAA{x{^R8B}QUV5j zaqBD0iSAbzMV+_f&wusbCjQwj6dEQMl)svi`c%^@X6EtJ+gJalR-h`%b*!j(dhb`+ zvWG?SNlvNOLe*kN__^GW#Mu4jhUqkc=uZ*i8&k2(X?pYf(lhW)F39?q9R(?1l3_`X%)QiT7s{HASGMWmRb z^oAzPqh5s-Alq#o)Ck;U90EJaE)r~6VtkTJKZArU^zHYsa^l22q%d?rPvv>^evtHmewrkH$V_LikY zI@4%b67vrE^ZtL+Qt(#wKPu|KNIw^~Au>^0KBHGYZC@Mp%n&TsVH#iOq-DRDDJnJKlMXMq+N3O zl9j)_M6B47GE)RW=Gf_*+2eF`Y-#u-(I9#*n>I0ge zaZa0@wK13J4Uc&@D?BCAW)K&$#06FQI0RJy0`W$j=q|3~=)r`efV1n6JjOgwmD3MA zeDL*shbbv|X6;Nez37>?=Jbf!vji%2{U3a(eGv*#=LCCY!l#P8;L#E9h-9-soCj$2&HnGm$ zJC9+bbW%H7@~kql6GiUY8FghK?VTK^dKp*eqA6|!TEu15@yGU;ujqoneTmyLat)e!cx;k6D6!_O)kFUG$?(Wx) zdZ%gzO?kMEduNz*VwBpUAUqc!d>Oz%G2pW*UrQ z69sZ*T75kN71i}i)Yp@55el)LF=6~`hQoKU__m}6+ZZ|7&q^?<)fZj_vJGs3&8UW! zw=`l`tGN*kTx<|kQ)rW=W)_Wg>5ILUrjZLdTX+zn<`mT>jH}kGTmV^;rEKQKll!=; zCB(pN09BP!&V&mJ=*C}2c){>Yu4NRTcx4fu9WBqN?2@dN2%X&^+En6WZzgCfGalEhVd>&rW}OFkW*N9{ zqxu26;V-tG;(lVa@X|LVN<`)(n0keog##X~ntV+A+2C?Tn4z`iDvB=@o&B zDdU%tEjrp2Ftxf{;pEX9**$|UgBQRzglTyrBGV#Y`W zIYv0ZUY|+P_uDFOfi9TwwI4D*OCh>Z>OSOx&zSG)P++ctS7`(wAm#9XTZjJd_VVw$ z`X@Rg*0|g#?PtnJJYWb!+VydRWJB|QpDFbiQ2-$sKpt8OmZg2rX_j%8gR4`m{j2s` z5!G{TCnL`kS7h2^&JAQsaaH(EEMO>|G1JEsWnnj0RT{@xnHfg05lvMe%F0Lk7u|zx z`y5GBnrGZxF70pZf;A*mKNP5WmN|@C?JeBud#7Hbj)XogY||94 zat0I)xqt2pNp6htZ?Wcb5O`y&CK)+5yeRmv%nC{-a5xfmA!U>LPdn9jVLS59gx4r`K#| zq48{;9@1>|)1v5AGfDc$%#;hNF3vl|ptmvIj8D7YEe1g`SHjB${0yu*0A|{!fAL@& zoYwNBLk63n`xrv_XdYgaZdM{YGRgC@1Pyn zoa7q9D@_^V^LCK7b$@Hka~LXwL7!qU277iQsx>}vLUkaPa)^_c5|v>fDESs>?&*Zi zQjsenz4vUORCk~vu_Obmv|P9?Fab?b@cP({7{ihi=ZYwtf_Zox)M-a>66RvKcv_ zX~?Mb<>>i>D;8;+1A{kc97X8R)s`ujRo&&(Dksb;kq7A~0#PS7CBwz-%9(wU-C@zN zSkYZ{4Tnm0YKhcXg~MwYut{y2rN?XO&t7haJI;Q1m4JImNshuC_)^|Db+bgC*Qfi3 zEv{urdhZrUC|DAWFmx^Chj?Wp?|}uvd;wJfT?in%Nv18uGKFUIPjl4iIwrr_vL?a1 zo)0twhD^mtbs9u^IubfU)CFNZPvGiUtqG`Oou*0#-WWRQ1H4%AAJQ7EAhQo>3c!3R zA%S&);ZFQ!z+nX`kP2%-=Azxn*6OheFd+QzUP}y|N zoy~y>5mE94B{#}tVBR-odRkt<5yE~65fO-qa7?seat;Lo47_Xd{LF!qxMs1AvclYK zzfS)&gwF#+;`6wpKMltwL-SpCoEpq78ZRFGnBeQqb8nvygFs;;!!>f3o@eX*8jhP? ze~pg+*tsOsGpgk1TC={K;t)n%Te?3bOq8#3jBxSZ*T@)#W8NY&c7K2swkQnX!WoM4 zg{7DH=n8Cq*#3Uo&)2l@Qxqf?Gz}yPg!7?om@x01;*YMlnXf`2C%}&*buKpgZU^LA zTuR>l$r#OOL=0O*5r9qU`YcjBiD(c#Jl}A!SK!6nnJ)LaVlIa+abzaA*stHeXchQ^ zRy%mm@IMuoD{+jXZGTktXW_XU^Xtux8C(PALI;h^Ep7r9afH&1)~rM^7Cr49$CVKh zvvSMcad1%}4F!MkMIOu6T}SF7k*E_dLV^!wr1f=75q311-O^Iq?xLLR*77qigCGO$ zx!q54S3s@Ab=L#+$`+7jMl0TS)i>JzX#u7;K_Pq=fzZ>#AFaU#!V67>x&UO3O2V~>+wjc7U$9gIwxx>GO5*$ayCWKNA^?y?&#%_e#_qNB^kmNg!qkK~YV z+=x@4h}mN31)qa=Fa3s*!q1!zRcqQ4V4t?=Dw z+0P_a<7OHx$%o0lqBoj1#|FUj5ax6weDESFWQ;A3%4bSTa6qxNDW-ku4v?^<& zAHwC!&DHXZHU1da3^6TRE`QrpGWRZ<$;k=xOVh#s2nB5mpS3=c;mJ-tZCM#0Du_whiPQh%`Jjvadoo zFi^W#kBKDVy#iJO>@Rn-C=4BxUVJcY;w@%L6rl5;=&HcO2_Q+P_ocCfth4R;B-=|> z7n^W==>AS$zHGBVqsGCtY}-gY>Rzs%mG)|@2nd7+1qLisBtptH zh6R2?L4{?sQ(rw>ge^^6H%>yh>*&XF=^rnrB-Jk7LfPh0fvXd}yu|F6JUa^6@2kr% z57SSD$LQ5057dJT!O6mVBxWpR8EDWw%E_X&8s1zK69fsV^~{KXmM_Q z9vXqJ^Hefgxt#3)#^IWKFmDiPvWZG`q%l)YHS6&DSzoZ|SY$u9{VwohDz8TeV@re6 zArA(5Tl;m50Vw+?>jV`|OYNI%`so}Ut#1+{jXxkA>5YZ75}XoFOc5;^vwk^G;SCqv zvD3khQ?*G4*&ZT7j_q(7+6g(d_n%>W6d1<7+ePiKVqxpJkveX=e7GRMPqyd5u(NaC zj=HgB*Z(9(Ox0LCo3Gwj?BZGb5y%K)J5Ev6*3IJVu{>u){d9RR($<13yNgW3!ij>R zeATSessO6+iM{VL;=`slQ+R3qZn;T!EPB2!X3ZA8uHi7XZG?yBbTlSq8|T?3_0(ct zx3&%?i?s?D(QbH<8sDc6>MC7@)@{khzdjx#ky(`$=#--+s3i2W^35{^wSMM?egRuu zeu}|CpPh8Yp39y`!=G}v5`R*}#zdKJpCnMl9%DjCJJX(Ey3hQW&4nXhy@jvO$9^i| zV26hugh5SUex=8RjFcM|-J9G(kF#nES~}WEWFzv8a`cPpaBMBibJEy|Fqghx1Q1XzFVU$sP5N}SE>~(vB=x}6lW3<63B{G zai;QeD)(u-TMEciL^0ixEH!4o`#xv%>7uITKG8;3d}j@ujY`Ms1%BPq5f8|?k@vx8 zlV7EGc;#59VT`0l_3Dt^GbTBkgUyb5OK$lC@ntGt%Ogk>!!-Yx& zXh(KM>ow%*@*GdX#xhqeHNM*Fn_!V;%sJV2-o3@V=dPFxa~6YO8SVBRdKHi1#<3dpqTmYMu;Ec#(45 zDW>t!gYRb2ubgyt>m^} z{Bdmy!|DyMi0?}WkW^>2r#D(um+HSQ9Z>%J3d`Eb%FxLEjhpgLd>Ba{j$C9y3%c`y z+2)T!Vu;Iy&MQS%YQa)HjA%ya0+wzUtp{aYYL_9*iJDO_-U2h6G6b25(bVMLYsfg0 z_jHVHc{K6&z#RZLWCIRJL%Jp!nMavW1<54Xd&au{Ds#Zn7tsyYDg=CLa&}s;tQ8Bs z{c%oNT7h|m;uF0QGdxI4Ik8k9JS~xGQ@UO3N2fvcuEYzYE=`C)1y*Q;T&8p|GBGr^ z;YzUhbh$M4UtmV*(22v|{R8nk+(VyOkB!e6aOac-s=kv&54me2*wLUjXyko3R$FSk zT|R0h z+3Q-mKHMOgb90mS2ZU%6jd~!eEp7>*KBGdpd2-B(3!2ud67$M9nUzM&2|yx31x%L|9x&notaVn#3CGrxgI&jen6c@xF5ISP65#!M?^Gcc zFtN`P)!r5j-V-EdmEFxT84S^674i}cFF005j3dKPv(ciYE+&j6r%^Qor|p0PVqTg0K6FOale-N%{AHg38&N>ZR^;@pXxCr2j)b0DD zT)&YQqm&oj{3Wb<5o0)9i1sxV13wERWeQ}AasBv@{;)6V!}(5;lScdh>JP>?){ZI$ zhL%R}{h=*h*5(hY)+4&&Xmw!UhvH44jWtGkqLaKhtk27f z^KOKDb#cKLYP40tfG@Hnk8K(-Y$+py-%*-$_|{gXy9i_uhHzxI5i}zuI1z6uEBkAp zmf#VOL=#uld#EGkf5(Xsc+a@Hqqj7qhaSWe8@)l3O@SVV)X=}Y!cs4dQ4E1HsumWr(+D=dOw77bsZ zUGCQ>c;92!umPo}}_lv7(Y8XINDgfJF9N$%~?cPkmrvA zVm)mi%2X{Daje*Rk1KnVc=M)X>x6(-QjONU9P8ZK7UHLRq+bMykVXWLhPV|AQr4Nz zA)vV8QDa!WL%OV6oQpaqAg=J#{im?lR07;C(b{N;OIRYz0Tj0Q0h3tV0Sa-kvl@uU zYz?e|3hW&9We5sr;G}G*+ZxK&*-_(Iv3VT1&O77?j&3z^4;Eh-BwKjf);K12$tSp5 z?ZOr-`5^Q)qm4CvLFV)WOOA+b4uX1YH*={!yiCf^Z5ZCb<1H}U`t{gZ$+t(_udT#= z<4e&b7pzqb}QBF0Y30Vh*xJ5ZKO<(?x`zl!?Cu!`)(H1AZ|D^g#i59Z#pdKf}ToJdY7A-W8fv52(+p7*J;^6fy2E67;*t7dJPjc zm37p z{MVTyTdN@WGLSuK2UfW@4p_2KJ!uC&594hSdn0Cy>`MDfe2ZyH%JFK5xT|$ai~@+O z{dD+l_p?5;qDH_eS`qH3sN?;Sr+bZBW_zTInpLJeFZ^*9~IGEW+{;#;_rEur=CKn|yCuqR>q(w;4MnLQhspm9A>%^z30G!}}f zb3y%46mqExwB#ndn57XF4Dr>;f8c;C*8+zlT(yOc%dkQ07EXLh2TOA$HcA;+zTQkX z#?rn3VQ&B(Qy|b`E8HDzFb-wMwDE-IBK<-*x9&QK4-jUWQAD;PmqsSP8B!@1qGnQ2 z2(ZVP0`Ak-fh?@&MXVj*G^Sq>?~yY{6u%8lh-uXO0eNH~eeSgDWaf??FQJF0jh)R3f#;?(;L zu;fj}E$Q}d3%Ar6uWA@?x)%#8nbywn`Me;1P@FBok!zXk6<&a8ddGCm(DOP{qv&CD z9H8Ff?hU@tOk}uaL8%MA(#ZJgdH96-?n(NP`s25y91zjJ_MP|Or7iL{$h?uT^;O(# zjU06U9BKIFdzg@d?q0$MXg@Bq5Pgkx7Bc*bU?g7QWk$b=xG*>Q;xTH5ODy#57jSsN zfXmrcI(}2SrA%BovFM3qpLk!i(We6CMFyH%9*?23g)e+mCfbP1d0M!%^`h5^=@Ly~ zq=p-O8jB^mOdtoWbd8vox^%v42*EBoJmH_10r6e^JFpx6YSFS{ohAm^hEMcDsBST8 ziTtNHC^d%NkK)pP+kiPWue==(r9(@LO{GUy5%7ptxnH)zCw4=F3B(G*JFh{JzLLSX zsMla8Dob%;*;p~gaFC}(Mo%%nQcW*a4R-M|)fSj-(wY0A$5pIO$P36(l?kRecN;l*ov{Hy3L;0jOJs zESAN}emn{Ex?Ef5h-b2gFH(5%tzP+ntaoQH_`09Vyg6g)jm)JW4Ge+?^w;+TzZ2)* zzaRL!{Ps1$KY_oKzTbKEe+lv1t^SQ=|4&W76O`Zk_Ft0u76<;&^gCtzpYY#lYVR!O zzl86NkNyYzpPc4@s`&j4!#khtFL8qTZwvmH-S$uL?`w>|H~$&yKj7bAN&FN3`;_c` zA@Y}Sk^cw&w`Ivc@xM05HwFJ08vlv@Jy87JdV6%gp+Em8%>NVq`=kB$)>kq7A7hJxH2B+m3J3`9?FaN0 KoCKKOul@%l)WTN) literal 0 HcmV?d00001 diff --git a/imports/api/account/methods.js b/imports/api/account/methods.js index 1c05cb9..f57e176 100644 --- a/imports/api/account/methods.js +++ b/imports/api/account/methods.js @@ -1,16 +1,27 @@ +import SimpleSchema from 'simpl-schema'; import { check } from 'meteor/check'; import { Meteor } from 'meteor/meteor'; import { Accounts } from 'meteor/accounts-base'; Meteor.methods({ + // add account addAccount(user) { - check(user, Object); + // validate user data + new SimpleSchema({ + email: { + type: String, + regEx: SimpleSchema.RegEx.Email, + }, + username: String, + password: String, + }).validate(user); const id = Accounts.createUser(user); if (!id) { throw new Meteor.Error('Error', 'Impossible de se connecter'); } }, + // delete account deleteAccount(id) { check(id, String); Meteor.users.remove({ _id: id }); diff --git a/imports/api/config/methods.js b/imports/api/config/methods.js new file mode 100644 index 0000000..7bed494 --- /dev/null +++ b/imports/api/config/methods.js @@ -0,0 +1,34 @@ + +import { check } from 'meteor/check'; +import { Meteor } from 'meteor/meteor'; +import configCollection from '../../collections/configCollection'; + +Meteor.methods({ + // define config + defineConfig(config) { + check(config, Object); + const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch(); + if (config.forbiddenWord) { + config.forbiddenWord = config.forbiddenWord.split(','); + } + if (config.domain) { + config.domain = config.domain.replace('https', 'http'); + } + // if config already exist, replace old value with new + if (oldConfig && oldConfig.length) { + configCollection.update(oldConfig[0]._id, { + $set: { + domain: config.domain || oldConfig.domain, + forbiddenWord: config.forbiddenWord || oldConfig.forbiddenWord, + }, + }); + } else { + // else insert + config.userId = Meteor.userId(); + configCollection.insert(config); + } + }, + getConfig() { + return configCollection.findOne({ userId: Meteor.userId() }); + }, +}); diff --git a/imports/api/crawl/crawlWebsite.js b/imports/api/crawl/crawlWebsite.js index 8ecc103..71611a5 100644 --- a/imports/api/crawl/crawlWebsite.js +++ b/imports/api/crawl/crawlWebsite.js @@ -10,11 +10,12 @@ export default class crawlWebsite { constructor(url) { this.urls = []; this.crawler = new Crawler(url); + let i = 0; this.crawler.on('fetchcomplete', (queueItem) => { const url = queueItem.url; + console.log(i++); if (checkData.checkURL(url)) { - console.log(url); this.urls.push(url); } }); diff --git a/imports/api/indexation/elasticSearchConfig.js b/imports/api/indexation/elasticSearchConfig.js index e6ee921..e454992 100644 --- a/imports/api/indexation/elasticSearchConfig.js +++ b/imports/api/indexation/elasticSearchConfig.js @@ -29,6 +29,7 @@ exports.analyser = { expand: true, synonyms: [ 'gosse, enfant', + 'pmi, protection maternelle et infantile', ], }, french_stemmer: { @@ -55,6 +56,9 @@ exports.analyser = { 'lowercase', ], }, + url_analyzer: { + tokenizer: 'uax_url_email', + }, }, }, }, @@ -107,7 +111,17 @@ exports.mapping = { }, url: { type: 'text', - analyzer: 'standard', + analyzer: 'url_analyzer', + }, + urlText: { + type: 'text', + analyzer: 'french_light', + fields: { + stemmed: { + type: 'text', + analyzer: 'french_heavy', + }, + }, }, h1: { type: 'text', diff --git a/imports/api/indexation/indexGeneric.js b/imports/api/indexation/indexGeneric.js index bfd69e3..227662e 100755 --- a/imports/api/indexation/indexGeneric.js +++ b/imports/api/indexation/indexGeneric.js @@ -64,6 +64,8 @@ export default class IndexGeneric { 'description.stemmed', 'body', 'body.stemmed', + 'urlText', + 'urlText.stemmed', 'url', ], }, diff --git a/imports/api/indexation/indexWebsite.js b/imports/api/indexation/indexWebsite.js index 036780b..7ff7b2c 100755 --- a/imports/api/indexation/indexWebsite.js +++ b/imports/api/indexation/indexWebsite.js @@ -1,4 +1,5 @@ +import url from 'url'; import { Meteor } from 'meteor/meteor'; import Sitemapper from 'sitemapper'; import Crawler from 'crawler'; @@ -12,23 +13,28 @@ export default class IndexWebsite extends IndexGeneric { /** * realize indexation - * @param url + * @param urlWebsite * @returns Promise */ - async initIndexation(url) { - const crawl = new CrawlWebsite(url); + async initIndexation(urlWebsite) { + this.config = { + domain: url.parse(urlWebsite).hostname, + forbiddenWord: [], + }; + this.config = await Meteor.call('getConfig', {}); - /* let urls = [url]; + console.log(this.config); + let urls = [urlWebsite]; // si c'est un sitemap on récupère ces url - if (checkData.isSitemap(url)) { - const { sites } = await sitemap.fetch(url); + if (checkData.isSitemap(urlWebsite)) { + const { sites } = await sitemap.fetch(urlWebsite); urls = sites; } return this.crawlUrl(_.uniq(urls)) .then(dataToIndex => this.indexByBulk(dataToIndex)) .catch((error) => { throw error; - });*/ + }); } /** @@ -44,18 +50,39 @@ export default class IndexWebsite extends IndexGeneric { this.listDataError = []; this.listDataForIndex = []; + this.listUrlAlreadyVisited = urls; + this.listPdf = []; console.log(`${urls.length} à parser!`); const crawl = new Crawler({ + skipDuplicates: true, callback: (error, res, done) => { if (error || res.statusCode !== 200) { this.listDataError.push({ url: res.options.uri, error: error || res.statusCode, }); - } else { + } else if (res && res.$) { + res.$('a').each((index, a) => { + const urlHref = res.$(a).attr('href'); + if (urlHref) { + const toQueueUrl = url.resolve(res.request.uri.href, res.$(a).attr('href')); + if (toQueueUrl.includes(this.config.domain) && !this.listUrlAlreadyVisited.includes(toQueueUrl)) { + if (!new RegExp(this.config.forbiddenWord.join('|')).test(toQueueUrl)) { + this.listUrlAlreadyVisited.push(toQueueUrl); + console.log(toQueueUrl); + crawl.queue(toQueueUrl); + } + } + } + }); this.parseData(res.$, res.options.uri); + } else { + if (!this.listUrlAlreadyVisited.includes(res.options.uri)) { + this.listUrlAlreadyVisited.push(res.options.uri); + this.listPdf.push(res.options.uri); + } } done(); @@ -65,6 +92,7 @@ export default class IndexWebsite extends IndexGeneric { crawl.queue(urls); crawl.on('drain', () => { console.error(this.listDataError); + console.log(this.listPdf); resolve(this.listDataForIndex); }); }); @@ -75,7 +103,7 @@ export default class IndexWebsite extends IndexGeneric { * @param $ * @param url */ - parseData($, url) { + parseData($, currentUrl) { const body = $('body'); body.html(checkData.cleanHtml(body.html())); const title = checkData.cleanText($('title').text()); @@ -88,7 +116,8 @@ export default class IndexWebsite extends IndexGeneric { description: $('meta[name=description]').attr('content'), body: checkData.slugText(checkData.cleanText(body.text())), html: checkData.cleanText(body.html()), - url: decodeURI(url), + urlText: checkData.slugText(checkData.cleanText(decodeURI(currentUrl))), + url: decodeURI(currentUrl), createdAt: new Date(), }; @@ -108,7 +137,7 @@ export default class IndexWebsite extends IndexGeneric { index: { _index: Meteor.settings.private.elasticsearch.esIndex, _type: Meteor.settings.private.elasticsearch.esType, - _id: url, + _id: currentUrl, }, }); diff --git a/imports/api/testSearch/methods.js b/imports/api/testSearch/methods.js index 5412f10..b54a000 100644 --- a/imports/api/testSearch/methods.js +++ b/imports/api/testSearch/methods.js @@ -1,13 +1,24 @@ -import { check } from 'meteor/check'; +import SimpleSchema from 'simpl-schema'; import { Meteor } from 'meteor/meteor'; import _ from 'lodash'; import testSearchCollection from '../../collections/testSearchCollection'; Meteor.methods({ addTest(test) { - check(test, Object); + // test data + new SimpleSchema({ + urlExpected: { + type: String, + regEx: SimpleSchema.RegEx.Url, + }, + term: String, + }).validate(test); + + // replace https by http test.urlExpected = _.replace(test.urlExpected, 'https', 'http'); + + // call search Meteor.call('searchByTerm', test.term, Meteor.bindEnvironment((err, results) => { if (err) { console.log(err); @@ -18,7 +29,16 @@ Meteor.methods({ test.urlPosition = index + 1; } }); - testSearchCollection.insert(test); + const oldTest = testSearchCollection.find({ term: test.term }).fetch(); + if (oldTest && oldTest.length) { + testSearchCollection.update(oldTest[0]._id, { + $set: { + urlPosition: test.urlPosition, + }, + }); + } else { + testSearchCollection.insert(test); + } } })); }, diff --git a/imports/collections/configCollection.js b/imports/collections/configCollection.js new file mode 100644 index 0000000..f18dd42 --- /dev/null +++ b/imports/collections/configCollection.js @@ -0,0 +1,29 @@ + +import SimpleSchema from 'simpl-schema'; +import { Mongo } from 'meteor/mongo'; + +/** + * this collection is for config (crawl, api, index) + * @type {Mongo.Collection} + */ +const configCollection = new Mongo.Collection('configs'); + +SimpleSchema.configCollection = new SimpleSchema({ + userId: { + type: String, + }, + domain: { + type: String, + regEx: SimpleSchema.RegEx.Url, + }, + forbiddenWord: { + type: Array, + }, + 'forbiddenWord.$': { + type: String, + }, +}); + +configCollection.attachSchema(SimpleSchema.configCollection); + +export default configCollection; diff --git a/imports/collections/testSearchCollection.js b/imports/collections/testSearchCollection.js index 24dcfa5..16fa7c9 100644 --- a/imports/collections/testSearchCollection.js +++ b/imports/collections/testSearchCollection.js @@ -3,7 +3,7 @@ import SimpleSchema from 'simpl-schema'; import { Mongo } from 'meteor/mongo'; /** - * this local collection keep tracks of all tabs of this application + * this collection is temporaire for testsearch * @type {Mongo.Collection} */ const testSearchCollection = new Mongo.Collection('testSearch'); diff --git a/imports/startup/server/register-api.js b/imports/startup/server/register-api.js index 85ff9d1..7de03c3 100755 --- a/imports/startup/server/register-api.js +++ b/imports/startup/server/register-api.js @@ -2,6 +2,7 @@ import '../../api/indexation/methods'; import '../../api/account/methods'; +import '../../api/config/methods'; import '../../api/testSearch/methods'; import '../../tabular/tabularUser'; import '../../tabular/tabularTestSearch'; diff --git a/imports/ui/components/testSearch/testSearch.html b/imports/ui/components/testSearch/testSearch.html index 844fbf9..542eaa4 100644 --- a/imports/ui/components/testSearch/testSearch.html +++ b/imports/ui/components/testSearch/testSearch.html @@ -17,6 +17,9 @@
{{> tabular table=TabularTables.testSearch class="table table-striped table-bordered table-condensed text-center"}}
+
+ +
diff --git a/imports/ui/components/testSearch/testSearch.js b/imports/ui/components/testSearch/testSearch.js index 15f5497..5b73345 100644 --- a/imports/ui/components/testSearch/testSearch.js +++ b/imports/ui/components/testSearch/testSearch.js @@ -20,6 +20,11 @@ Template.testSearchTpl.events({ console.log(error); }); }, + 'click .launch-test': (event) => { + event.preventDefault(); + + + }, }); Template.testSearchActionTable.events({ diff --git a/imports/ui/pages/admin/account/add/add.js b/imports/ui/pages/admin/account/add/add.js index 2933fb2..fee6cc2 100644 --- a/imports/ui/pages/admin/account/add/add.js +++ b/imports/ui/pages/admin/account/add/add.js @@ -6,6 +6,7 @@ import displayNotif from '../../../../components/notifs/notifs'; import './add.html'; Template.addAccountTpl.events({ + // add account 'submit form': (event) => { event.preventDefault(); diff --git a/imports/ui/pages/admin/indexation/api/api.html b/imports/ui/pages/admin/indexation/api/api.html index aa00770..bfc7736 100755 --- a/imports/ui/pages/admin/indexation/api/api.html +++ b/imports/ui/pages/admin/indexation/api/api.html @@ -1,7 +1,7 @@ diff --git a/imports/ui/pages/admin/indexation/site/site.js b/imports/ui/pages/admin/indexation/site/site.js index 697b247..4a00514 100755 --- a/imports/ui/pages/admin/indexation/site/site.js +++ b/imports/ui/pages/admin/indexation/site/site.js @@ -7,6 +7,20 @@ import displayNotif from '../../../../components/notifs/notifs.js'; import './site.html'; +Template.siteIndexationTpl.onRendered(() => { + $('.forbiddenWordWebsite').tagsinput(); + Meteor.callPromise('getConfig', {}) + .then((config) => { + $('.hostUrlWebsite').val(config.domain); + if (config.forbiddenWord && config.forbiddenWord.length) { + config.forbiddenWord = config.forbiddenWord.join(','); + $('.forbiddenWordWebsite').tagsinput('add', config.forbiddenWord); + } + $('.bootstrap-tagsinput').addClass('form-control'); + }); + +}); + Template.siteIndexationTpl.events({ 'submit #formUrlSite': (event) => { event.preventDefault(); @@ -42,4 +56,31 @@ Template.siteIndexationTpl.events({ }); event.target.reset(); }, + 'submit #formReglageSite': (event) => { + event.preventDefault(); + + const config = { + domain: event.target.hostUrlWebsite.value, + forbiddenWord: event.target.forbiddenWordWebsite.value, + }; + + Meteor.callPromise('defineConfig', config) + .then(() => { + displayNotif({ + type: 'success', + title: 'Configuration : ', + message: 'Configuration ajouté avec succès!', + save: true, + }); + }).catch((error) => { + displayNotif({ + type: 'error', + title: 'Configuration : ', + message: error.reason ? error.reason : error, + save: true, + }); + }); + event.target.reset(); + $('.forbiddenWordWebsite').tagsinput('removeAll'); + }, }); diff --git a/imports/ui/pages/login/login.js b/imports/ui/pages/login/login.js index bf2ae32..8f10c54 100755 --- a/imports/ui/pages/login/login.js +++ b/imports/ui/pages/login/login.js @@ -1,5 +1,6 @@ import { Meteor } from 'meteor/meteor'; +import { Session } from 'meteor/session'; import { FlowRouter } from 'meteor/kadira:flow-router'; import { Template } from 'meteor/templating'; import displayNotif from '../../components/notifs/notifs'; @@ -29,6 +30,12 @@ Template.loginTpl.events({ title: 'Succès : ', message: `Bienvenue ${user.login}!`, }); + // get global config for user + Meteor.callPromise('getConfig', {}) + .then((config) => { + console.log(config); + Session.set('config', config); + }); } }); }, diff --git a/package.json b/package.json index ee12623..5002cbd 100755 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "cheerio": "^1.0.0-rc.1", "crawler": "^1.0.5", "datatables.net-bs": "^1.10.15", - "elasticsearch": "^13.1.0", + "elasticsearch": "^13.1.1", "izitoast": "^1.1.1", "jquery": "^1.11.2", "lodash": "^4.17.4", diff --git a/yarn.lock b/yarn.lock index e6b4b3d..42c7fab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -896,20 +896,13 @@ domhandler@^2.3.0: dependencies: domelementtype "1" -domutils@1.5.1: +domutils@1.5.1, domutils@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" dependencies: dom-serializer "0" domelementtype "1" -domutils@^1.5.1: - version "1.6.2" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.6.2.tgz#1958cc0b4c9426e9ed367fb1c8e854891b0fa3ff" - dependencies: - dom-serializer "0" - domelementtype "1" - double-ended-queue@^2.1.0-0: version "2.1.0-0" resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" @@ -924,9 +917,9 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" -elasticsearch@^13.1.0: - version "13.1.0" - resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-13.1.0.tgz#f22a2afe01f4c3cde660bdd5b77575c6f8b6a397" +elasticsearch@^13.1.1: + version "13.1.1" + resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-13.1.1.tgz#83dbb5cf76ecc9bdd3ffceb5ce74d5fc7d23e798" dependencies: agentkeepalive "^2.2.0" chalk "^1.0.0" -- 2.22.0