Commit c2422588 authored by Nacim Goura's avatar Nacim Goura

add autocompletion and fix bug

parent 148d1046
...@@ -48,7 +48,7 @@ html-tools@1.0.11 ...@@ -48,7 +48,7 @@ html-tools@1.0.11
htmljs@1.0.11 htmljs@1.0.11
http@1.2.12 http@1.2.12
id-map@1.0.9 id-map@1.0.9
johanbrook:publication-collector@1.0.9 johanbrook:publication-collector@1.0.10
jquery@1.11.10 jquery@1.11.10
kadira:blaze-layout@2.3.0 kadira:blaze-layout@2.3.0
kadira:flow-router@2.12.1 kadira:flow-router@2.12.1
......
...@@ -48,7 +48,7 @@ export default class crawlWebsite extends CrawlGeneric { ...@@ -48,7 +48,7 @@ export default class crawlWebsite extends CrawlGeneric {
console.log(`${urls.length} à parser!`); console.log(`${urls.length} à parser!`);
const crawl = new Crawler({ const crawl = new Crawler({
retries: 5, retries: 10,
skipDuplicates: true, skipDuplicates: true,
rotateUA: true, rotateUA: true,
userAgent: [ userAgent: [
...@@ -141,7 +141,8 @@ export default class crawlWebsite extends CrawlGeneric { ...@@ -141,7 +141,8 @@ export default class crawlWebsite extends CrawlGeneric {
domain: this.config.domain, domain: this.config.domain,
title, title,
title_suggest: { title_suggest: {
input: title, // replace - and _ and multiple space for autocompletion
input: title.replace(/[-_]/g, ' ').replace(/[^\S]{2,}/, ' ').trim(),
}, },
description: $('meta[name=description]').attr('content'), description: $('meta[name=description]').attr('content'),
body: checkData.cleanText(body.text()), body: checkData.cleanText(body.text()),
......
const Crawler = require('simplecrawler');
const crawler = new Crawler('http://www.anap.fr/');
crawler.on('fetchcomplete', (queueItem, data, res) => {
console.log(queueItem.url);
});
crawler.start();
...@@ -52,7 +52,7 @@ export async function searchDocument(data, userId) { ...@@ -52,7 +52,7 @@ export async function searchDocument(data, userId) {
} }
export async function searchAll(data) { export async function searchAll(data) {
const userId = Meteor.userId().toLowerCase(); const userId = Meteor.userId();
try { try {
return { return {
website: await searchWebsite(data, userId), website: await searchWebsite(data, userId),
......
...@@ -5,8 +5,9 @@ import indexationElastic from '/imports/libs/elasticsearch/elasticsearch'; ...@@ -5,8 +5,9 @@ import indexationElastic from '/imports/libs/elasticsearch/elasticsearch';
export default class Search { export default class Search {
constructor(userId) { constructor(userId, config) {
this.esIndex = userId; this.esIndex = userId.toLowerCase();
this.config = config;
} }
/** /**
...@@ -29,15 +30,22 @@ export default class Search { ...@@ -29,15 +30,22 @@ export default class Search {
params._source = ["title", "url"]; params._source = ["title", "url"];
params.query = { params.query = {
"bool": { "bool": {
"must_not": [
{
"match_phrase": {
"url": "page",
},
},
],
"must": [ "must": [
{ {
"multi_match": { "multi_match": {
"query": term, "query": term,
"fuzziness": "AUTO",
"fields": [ "fields": [
"title.stemmed", "title.stemmed",
"urlText.stemmed", "urlText.stemmed",
"description.stemmed", "description.stemmed",
"url",
], ],
"boost": 3, "boost": 3,
}, },
...@@ -47,15 +55,12 @@ export default class Search { ...@@ -47,15 +55,12 @@ export default class Search {
{ {
"multi_match": { "multi_match": {
"query": term, "query": term,
"fuzziness": "AUTO",
"fields": [ "fields": [
"title",
"description", "description",
"urlText", "urlText",
"title",
"h1", "h1",
"breadcrumb", "breadcrumb",
"urlText",
"urlText.stemmed",
], ],
"boost": 2, "boost": 2,
}, },
...@@ -63,18 +68,18 @@ export default class Search { ...@@ -63,18 +68,18 @@ export default class Search {
{ {
"multi_match": { "multi_match": {
"query": term, "query": term,
"fuzziness": "AUTO",
"fields": [ "fields": [
"body.stemmed", "body.stemmed",
"body", "body",
"h2", "h2",
"html", "html",
"listPdf",
], ],
"boost": 1, "boost": 1,
}, },
}, },
], ],
"minimum_should_match": 1, "minimum_should_match": 2,
}, },
}; };
} }
...@@ -89,6 +94,7 @@ export default class Search { ...@@ -89,6 +94,7 @@ export default class Search {
}; };
if (term) { if (term) {
params._source = ["title", "url"];
params.query = { params.query = {
bool: { bool: {
must: [{ must: [{
...@@ -181,15 +187,16 @@ export default class Search { ...@@ -181,15 +187,16 @@ export default class Search {
* @returns {promise} * @returns {promise}
*/ */
autoComplete(term) { autoComplete(term) {
let params = {}; const params = {};
if (term) { if (term) {
params = { params.suggest = {
suggest: { text: term,
prefix: term, completion: {
completion: { field: 'title_suggest',
field: 'title_suggest', size: 25,
size: 25, fuzzy: {
fuzziness: 2,
}, },
}, },
}; };
......
...@@ -12,14 +12,17 @@ export function addTest(newTest) { ...@@ -12,14 +12,17 @@ export function addTest(newTest) {
if (!newTest.userId) { if (!newTest.userId) {
newTest.userId = Meteor.userId(); newTest.userId = Meteor.userId();
} }
newTest.searchTerm = newTest.searchTerm.trim();
newTest.createdAt = moment().toDate(); newTest.createdAt = moment().toDate();
testSearchCollection.simpleSchema().validate(newTest); testSearchCollection.simpleSchema().validate(newTest);
newTest.urlExpected = newTest.urlExpected.replace('https', 'http').replace(/\/$/, '').trim();
// call search // call search
searchWebsite({ searchTerm: newTest.searchTerm }) searchWebsite({ searchTerm: newTest.searchTerm }, newTest.userId)
.then((results) => { .then((results) => {
_.forEach(results.list, (result, index) => { _.forEach(results.list, (result, index) => {
if (newTest.urlExpected === result.url) { if (newTest.urlExpected.toLowerCase().trim() === result.url.toLowerCase().trim()) {
newTest.urlPosition = index + 1; newTest.urlPosition = index + 1;
} }
}); });
......
...@@ -9,19 +9,9 @@ exports.analyser = { ...@@ -9,19 +9,9 @@ exports.analyser = {
type: 'elision', type: 'elision',
articles_case: true, articles_case: true,
articles: [ articles: [
'l', 'l', 'm', 't', 'qu', 'n', 's',
'm', 'j', 'd', 'c', 'jusqu', 'quoiqu',
't', 'lorsqu', 'puisqu',
'qu',
'n',
's',
'j',
'd',
'c',
'jusqu',
'quoiqu',
'lorsqu',
'puisqu',
], ],
}, },
// synonyme // synonyme
...@@ -30,8 +20,13 @@ exports.analyser = { ...@@ -30,8 +20,13 @@ exports.analyser = {
ignore_case: true, ignore_case: true,
expand: true, expand: true,
synonyms: [ synonyms: [
'gosse, enfant',
'pmi, protection maternelle et infantile', 'pmi, protection maternelle et infantile',
'psycho, psychiatrie',
'anap, Agence nationale d\'appui à la performance des établissements de santé et médico-sociaux',
'ars, agences régionales de santé',
'c dans l\'oxygene, c dans l\'air',
'+, plus',
'%, pour cent',
], ],
}, },
// radical des mots // radical des mots
...@@ -39,6 +34,18 @@ exports.analyser = { ...@@ -39,6 +34,18 @@ exports.analyser = {
type: 'stemmer', type: 'stemmer',
language: 'light_french', language: 'light_french',
}, },
// phonetic (alternative a fuziness pour les erreurs d'orthographe)
phonetic_filter: {
type: 'phonetic',
encoder: 'metaphone',
replace: false,
},
},
tokenizer: {
tokeniser_url_mail: {
type: 'uax_url_email',
max_token_length: 5,
},
}, },
analyzer: { analyzer: {
// français elevé // français elevé
...@@ -50,7 +57,7 @@ exports.analyser = { ...@@ -50,7 +57,7 @@ exports.analyser = {
'french_synonym', 'french_synonym',
'french_stemmer', 'french_stemmer',
'lowercase', 'lowercase',
'asciifolding', 'phonetic_filter',
], ],
}, },
// français léger // français léger
...@@ -63,16 +70,19 @@ exports.analyser = { ...@@ -63,16 +70,19 @@ exports.analyser = {
'french_elision', 'french_elision',
'icu_folding', 'icu_folding',
'lowercase', 'lowercase',
'asciifolding', 'phonetic_filter',
], ],
}, },
// analyzer for url // analyzer for url
url_analyzer: { url_analyzer: {
tokenizer: 'uax_url_email', tokenizer: 'tokeniser_url_mail',
filter: [ filter: [
'french_synonym',
'french_stemmer',
'french_elision', 'french_elision',
'lowercase',
'icu_folding', 'icu_folding',
'lowercase',
'phonetic_filter',
], ],
}, },
}, },
......
...@@ -65,3 +65,14 @@ adminSection.route('/notif', { ...@@ -65,3 +65,14 @@ adminSection.route('/notif', {
}); });
}, },
}); });
adminSection.route('/test', {
name: 'testSearch',
action() {
BlazeLayout.render('mainLayoutTpl', {
sidebar: 'sidebarLayoutTpl',
main: 'testSearchTpl',
navbar: 'navbarLayoutTpl',
});
},
});
...@@ -18,6 +18,7 @@ import '/imports/ui/pages/admin/indexation/indexation'; ...@@ -18,6 +18,7 @@ import '/imports/ui/pages/admin/indexation/indexation';
import '/imports/ui/pages/admin/account/account'; import '/imports/ui/pages/admin/account/account';
import '/imports/ui/pages/admin/statistique/statistique'; import '/imports/ui/pages/admin/statistique/statistique';
import '/imports/ui/pages/notif/notif'; import '/imports/ui/pages/notif/notif';
import '/imports/ui/components/testSearch/testSearch';
// Set up all routes in the app // Set up all routes in the app
import './adminRoutes'; import './adminRoutes';
......
...@@ -19,4 +19,6 @@ SyncedCron.add({ ...@@ -19,4 +19,6 @@ SyncedCron.add({
}, },
}); });
SyncedCron.start(); if (Meteor.isProduction) {
SyncedCron.start();
}
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
</button> </button>
</span> </span>
</div> </div>
<!--
{{#if listAutoCompleteResults}} {{#if listAutoCompleteResults}}
<div class="autocomplete-dropdown"> <div class="autocomplete-dropdown">
{{#each result in listAutoCompleteResults}} {{#each result in listAutoCompleteResults}}
...@@ -22,7 +21,6 @@ ...@@ -22,7 +21,6 @@
{{/each}} {{/each}}
</div> </div>
{{/if}} {{/if}}
-->
{{/autoForm}} {{/autoForm}}
</div> </div>
......
...@@ -14,23 +14,25 @@ Template.searchTpl.helpers({ ...@@ -14,23 +14,25 @@ Template.searchTpl.helpers({
}); });
Template.searchTpl.events({ Template.searchTpl.events({
/* 'input #searchResult': (event) => { 'input input[name$=searchTerm]': (event) => {
event.preventDefault(); event.preventDefault();
const term = event.target.value; const term = event.target.value;
Meteor.callPromise('autoCompletion', term) if (term && term.length > 2) {
.then((results) => { Meteor.callPromise('autoCompletion', term)
Session.set('autoCompleteResults', results); .then((results) => {
}); Session.set('autoCompleteResults', results);
}, */ });
}
},
'click .autocomplete-title': (event) => { 'click .autocomplete-title': (event) => {
event.preventDefault(); event.preventDefault();
const term = event.target.textContent; const term = event.target.textContent;
$('#searchResult').val(term); $('input[name$=searchTerm]').val(term);
Session.set('autoCompleteResults', null); Session.set('autoCompleteResults', null);
}, },
......
...@@ -10,12 +10,15 @@ ...@@ -10,12 +10,15 @@
<li> <li>
<a href="/admin"><i class="fa fa-fw fa-etsy fa-20px" aria-hidden="true"></i> Indexation</a> <a href="/admin"><i class="fa fa-fw fa-etsy fa-20px" aria-hidden="true"></i> Indexation</a>
</li> </li>
<li>
<a href="/admin/test"><i class="fa fa-fw fa-gear fa-20px" aria-hidden="true"></i> Tests</a>
</li>
{{#if grantedAccessAdmin}} {{#if grantedAccessAdmin}}
<li> <li>
<a href="/admin/account"><i class="fa fa-fw fa-user fa-20px" aria-hidden="true"></i> Comptes</a> <a href="/admin/account"><i class="fa fa-fw fa-user fa-20px" aria-hidden="true"></i> Comptes</a>
</li> </li>
<li> <li>
<a href="/admin/stat"><i class="fa fa-fw fa-bar-chart-o fa-20px" aria-hidden="true"></i> Statistiques</a> <a href="/admin/stat"><i class="fa fa-fw fa-bar-chart-o fa-20px" aria-hidden="true"></i> Statistiques</a>
</li> </li>
{{/if}} {{/if}}
<li> <li>
......
...@@ -5,10 +5,22 @@ ...@@ -5,10 +5,22 @@
<div class="panel panel-default wrapper"> <div class="panel panel-default wrapper">
<div class="panel-body"> <div class="panel-body">
<h4 class="text-center">Gestion de l'indexation en général</h4> <h4 class="text-center">Gestion de l'indexation en général</h4>
<button class="btn btn-danger" id="initElastic">Initialisation ElasticSearch</button> <div class="row">
<!--<button class="btn btn-primary btn-reindex" id="websiteReindexation">Réindexer sites web</button> <button class="btn btn-danger" id="initElastic">Initialisation ElasticSearch</button>
<button class="btn btn-primary btn-reindex" id="apiReindexation">Réindexer API</button> <!--<button class="btn btn-primary btn-reindex" id="websiteReindexation">Réindexer sites web</button>
<button class="btn btn-primary btn-reindex" id="documentReindexation">Réindexer documents</button>--> <button class="btn btn-primary btn-reindex" id="apiReindexation">Réindexer API</button>
<button class="btn btn-primary btn-reindex" id="documentReindexation">Réindexer documents</button>-->
</div>
<hr>
<div class="row">
<label>Liste des synonymes : </label>
<textarea class="form-control" name="synonym" id="" cols="30" rows="10"></textarea>
<div class="alert alert-warning">
<strong>Attention!</strong> Changer les synonymes nécessite une réindexation!
</div>
</div>
</div> </div>
</div> </div>
</template> </template>
...@@ -6,6 +6,10 @@ import displayNotif from '/imports/api/notif/notifClient'; ...@@ -6,6 +6,10 @@ import displayNotif from '/imports/api/notif/notifClient';
import './main.html'; import './main.html';
Template.mainIndexationTpl.hooks({
rendered() {},
});
Template.mainIndexationTpl.events({ Template.mainIndexationTpl.events({
'click #initElastic': (event) => { 'click #initElastic': (event) => {
event.preventDefault(); event.preventDefault();
...@@ -33,7 +37,7 @@ Template.mainIndexationTpl.events({ ...@@ -33,7 +37,7 @@ Template.mainIndexationTpl.events({
message: error.message, message: error.message,
save: true, save: true,
}); });
}); });
}); });
}, },
'click .btn-reindex': (event) => { 'click .btn-reindex': (event) => {
......
...@@ -4,6 +4,4 @@ ...@@ -4,6 +4,4 @@
{{> searchTpl ""}} {{> searchTpl ""}}
{{> testSearchTpl ""}}
</template> </template>
...@@ -70,7 +70,7 @@ const checkData = { ...@@ -70,7 +70,7 @@ const checkData = {
// url for crawl // url for crawl
checkCrawlUrl(url) { checkCrawlUrl(url) {
return !/\.(jpeg|jpg|gif|png|js|css|ico|eot|svg|woff|ttf|zip|rar|tar|pdf|xml|xlsx|docx)/.test(url); return !/\.(jpeg|jpg|gif|png|js|css|ico|eot|svg|woff|ttf|zip|rar|tar|pdf|xml|xlsx|docx|mp3)/.test(url);
}, },
// clean url // clean url
......
pmi, protection maternelle et infantile
psycho, psychiatrie
anap, Agence nationale d\'appui à la performance des établissements de santé et médico-sociaux
ars, agences régionales de santé
c dans l'oxygene, c dans l'air
+, plus
%, pour cent
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment