Commit 7a2af955 authored by Nacim Goura's avatar Nacim Goura

aadd readme, config file and optimisation

parent 6dab519a
...@@ -18,7 +18,7 @@ standard-minifier-js@2.1.0 # JS minifier run for production mode ...@@ -18,7 +18,7 @@ standard-minifier-js@2.1.0 # JS minifier run for production mode
es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers. es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers.
ecmascript@0.8.0 # Enable ECMAScript2015+ syntax in app code ecmascript@0.8.0 # Enable ECMAScript2015+ syntax in app code
fourseven:scss # Compile scss to css fourseven:scss # Compile scss to css
fortawesome:fontawesome # icon fortawesome:fontawesome # pretty icon
# package for view # package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor kadira:flow-router # FlowRouter is a very simple router for Meteor
......
# Project meteorSearch
Application développée sous Meteor 1.5
## Installation locale
Installation du framework Meteor
```
$ curl https://install.meteor.com/ | sh
```
Création du projet et lancement du serveur local
```
$ mkdir meteorSearch && cd meteorSearch
$ git clone git@gitlab.idfr.net:IDsearch/IDsearch.git ./
$ meteor npm install
$ meteor
```
Chargement de l'application dans le navigateur à l'url: `http://localhost:3000`
## Deploiement sur un serveur distant
Le déploiement s'effectue à l'aide du module pm2-meteor
### Préparation du serveur
Installation de node version 6.x
```
$ curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
$ sudo apt-get install -y nodejs
```
Installation de nginx
```
$ sudo apt-get install nginx
$ systemctl status nginx
```
Configuration de nginx
```
$ cd /etc/nginx/sites-available
$ sudo cp default meteorSearch
$ sudo vim meteorSearch
```
Ajouter ou completer les lignes suivantes du fichier:
```
server_tokens off; # for security-by-obscurity: stop displaying nginx version
# this section is needed to proxy web-socket connections
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server_name -url-application-; # the domain on which we want to host the application. Since we set "default_server" previously, nginx will answer all hosts anyway.
# pass all requests to Meteor
location / {
proxy_pass http://127.0.0.1:3000;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade; # allow websockets
proxy_set_header Connection $connection_upgrade;
proxy_set_header X-Forwarded-For $remote_addr; # preserve client IP
# this setting allows the browser to cache the application in a way compatible with Meteor
# on every applicaiton update the name of CSS and JS file is different, so they can be cache infinitely (here: 30 days)
# the root path (/) MUST NOT be cached
if ($uri != '/') {
expires 30d;
}
}
```
Activation du virtualhost
```
$ sudo rm /etc/nginx/sites-enabled/default
$ sudo ln -s /etc/nginx/sites-available/meteorSearch /etc/nginx/sites-enabled/meteorSearch
$ sudo nginx -t
$ sudo nginx -s reload
```
Installation de MongoDB
```
$ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
$ echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
$ sudo apt-get update
$ sudo apt-get install -y mongodb-org
```
Création du service MongoDB
```
$ sudo vim /etc/systemd/system/mongodb.service
```
Installation de Java
```
sudo add-apt-repository -y ppa:webupd8team/java && sudo apt-get update && sudo apt-get install -y oracle-java8-installer
```
Installation d'elasticSearch
```
$ curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.4.1.tar.gz
$ tar -xvf elasticsearch-5.4.1.tar.gz
$ cd elasticsearch-5.4.1/bin
$ ./elasticsearch
```
Création du service MongoDB
```
$ sudo vim /etc/systemd/system/mongodb.service
```
Coller le contenu suivant:
```
[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
[Service]
User=mongodb
ExecStart=/usr/bin/mongod --quiet --config /etc/mongod.conf
[Install]
WantedBy=multi-user.target
```
Démarrage du service
```
$ sudo systemctl start mongodb
$ sudo systemctl status mongodb
$ sudo systemctl enable mongodb
```
Création et configuration du dossier de déploiement
```
$ sudo mkdir /opt/pm2-meteor/meteorSearch
$ sudo chown -R www-data:www-data /opt/pm2-meteor/meteorSearch/
```
## Configuration locale
Installation de pm2-meteor et initialisation
```
$ npm i pm2-meteor -g
$ cd meteorSearch
$ mkdir .deploy && cd .deploy
$ pm2-meteor init
```
Ouvrir le fichier /.deploy/pm2-meteor.json et recopier en adaptant:
```
{
"appName": "meteorSearch",
"appLocation": {
"local": "/home/ngoura/Documents/projet/meteorSearch"
},
"meteorSettingsLocation": "/home/ngoura/Documents/projet/meteorSearch/settings.json",
"meteorSettingsInRepo": false,
"prebuildScript": "",
"meteorBuildFlags": "--architecture os.linux.x86_64",
"env": {
"ROOT_URL": "http://idsearch.idfr.net",
"PORT": "3000",
"MONGO_URL": "mongodb://localhost:27017/meteorSearch"
},
"server": {
"host": "92.243.12.112",
"username": "www-data",
"password": "...",
"deploymentDir": "/opt/pm2-meteor",
"loadProfile": "",
"nvm": {
"bin": "",
"use": ""
},
"exec_mode": "cluster_mode",
"instances": 1
}
}
```
Déployer et lancer l'application:
```
$ pm2-meteor deploy
```
// analyser
exports.analyser = {
settings: {
analysis: {
filter: {
french_elision: {
type: 'elision',
articles_case: true,
articles: [
'l',
'm',
't',
'qu',
'n',
's',
'j',
'd',
'c',
'jusqu',
'quoiqu',
'lorsqu',
'puisqu',
],
},
french_synonym: {
type: 'synonym',
ignore_case: true,
expand: true,
synonyms: [
'gosse, enfant',
],
},
french_stemmer: {
type: 'stemmer',
language: 'light_french',
},
},
analyzer: {
french_heavy: {
tokenizer: 'icu_tokenizer',
filter: [
'french_elision',
'icu_folding',
'french_synonym',
'french_stemmer',
'lowercase',
],
},
french_light: {
tokenizer: 'icu_tokenizer',
filter: [
'french_elision',
'icu_folding',
'lowercase',
],
},
},
},
},
};
// mapping
exports.mapping = {
properties: {
tag: {
type: 'keyword',
},
title: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
title_suggest: {
type: 'completion',
analyzer: 'french_light',
max_input_length: 100,
},
description: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
body: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
html: {
type: 'text',
analyzer: 'standard',
},
url: {
type: 'text',
analyzer: 'standard',
},
h1: {
type: 'text',
analyzer: 'french_light',
},
h2: {
type: 'text',
analyzer: 'french_light',
},
breadcrumb: {
type: 'text',
analyzer: 'french_light',
},
createdAt: {
type: 'date',
},
},
};
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import indexationElastic from '../../libs/elasticsearch/elasticsearch'; import indexationElastic from '../../libs/elasticsearch/elasticsearch';
import configElastic from './elasticSearchConfig';
const esIndex = Meteor.settings.private.elasticsearch.esIndex; const esIndex = Meteor.settings.private.elasticsearch.esIndex;
const esType = Meteor.settings.private.elasticsearch.esType; const esType = Meteor.settings.private.elasticsearch.esType;
...@@ -26,117 +27,9 @@ export default class IndexGeneric { ...@@ -26,117 +27,9 @@ export default class IndexGeneric {
* init analyser and mapping * init analyser and mapping
*/ */
initElastic() { initElastic() {
const analyser = { const analyser = configElastic.analyser;
settings: {
analysis: {
filter: {
french_elision: {
type: 'elision',
articles_case: true,
articles: [
'l', 'm', 't', 'qu', 'n', 's',
'j', 'd', 'c', 'jusqu', 'quoiqu',
'lorsqu', 'puisqu',
],
},
french_synonym: {
type: 'synonym',
ignore_case: true,
expand: true,
synonyms: ['gosse, enfant'],
},
french_stemmer: {
type: 'stemmer',
language: 'light_french',
},
},
analyzer: {
french_heavy: {
tokenizer: 'icu_tokenizer',
filter: [
'french_elision',
'icu_folding',
'french_synonym',
'french_stemmer',
],
},
french_light: {
tokenizer: 'icu_tokenizer',
filter: [
'french_elision',
'icu_folding',
],
},
},
},
},
};
const mapping = { const mapping = configElastic.mapping;
properties: {
tag: {
type: 'keyword',
},
title: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
title_suggest: {
type: 'completion',
analyzer: 'french_light',
max_input_length: 100,
},
description: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
body: {
type: 'text',
analyzer: 'french_light',
fields: {
stemmed: {
type: 'text',
analyzer: 'french_heavy',
},
},
},
html: {
type: 'text',
analyzer: 'standard',
},
url: {
type: 'text',
analyzer: 'standard',
},
h1: {
type: 'text',
analyzer: 'french_light',
},
h2: {
type: 'text',
analyzer: 'french_light',
},
breadcrumb: {
type: 'text',
analyzer: 'french_light',
},
createdAt: {
type: 'date',
},
},
};
return indexationElastic.initAnalyzer(esIndex, analyser) return indexationElastic.initAnalyzer(esIndex, analyser)
.then(() => indexationElastic.initMapping(esIndex, esType, mapping)); .then(() => indexationElastic.initMapping(esIndex, esType, mapping));
...@@ -162,13 +55,6 @@ export default class IndexGeneric { ...@@ -162,13 +55,6 @@ export default class IndexGeneric {
params.query = { params.query = {
bool: { bool: {
must: [ must: [
{
common: {
'body.stemmed': {
query: term,
},
},
},
{ {
multi_match: { multi_match: {
query: term, query: term,
...@@ -182,7 +68,6 @@ export default class IndexGeneric { ...@@ -182,7 +68,6 @@ export default class IndexGeneric {
], ],
}, },
}, },
], ],
should: [ should: [
{ {
......
...@@ -21,7 +21,7 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -21,7 +21,7 @@ export default class IndexWebsite extends IndexGeneric {
const { sites } = await sitemap.fetch(url); const { sites } = await sitemap.fetch(url);
urls = sites; urls = sites;
} }
return this.crawlUrl(urls) return this.crawlUrl(_.uniq(urls))
.then(dataToIndex => this.indexByBulk(dataToIndex)) .then(dataToIndex => this.indexByBulk(dataToIndex))
.catch((error) => { .catch((error) => {
throw error; throw error;
...@@ -42,21 +42,16 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -42,21 +42,16 @@ export default class IndexWebsite extends IndexGeneric {
this.listDataError = []; this.listDataError = [];
this.listDataForIndex = []; this.listDataForIndex = [];
console.log(`${urls.length} à parser!`);
const crawl = new Crawler({ const crawl = new Crawler({
callback: (error, res, done) => { callback: (error, res, done) => {
if (error) { if (error || res.statusCode !== 200) {
this.listDataError.push({ this.listDataError.push({
url: res.options.uri, url: res.options.uri,
error, error: error || res.statusCode,
}); });
} else { } else {
if (res.statusCode !== 200) {
this.listDataError.push({
url: res.options.uri,
error: res.statusCode,
});
}
this.parseData(res.$, res.options.uri); this.parseData(res.$, res.options.uri);
} }
...@@ -80,7 +75,7 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -80,7 +75,7 @@ export default class IndexWebsite extends IndexGeneric {
parseData($, url) { parseData($, url) {
const body = $('body'); const body = $('body');
body.html(checkData.cleanHtml(body.html())); body.html(checkData.cleanHtml(body.html()));
const title = $('title').text(); const title = checkData.cleanText($('title').text());
const dataForIndex = { const dataForIndex = {
tag: 'site', tag: 'site',
title, title,
...@@ -94,10 +89,8 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -94,10 +89,8 @@ export default class IndexWebsite extends IndexGeneric {
createdAt: new Date(), createdAt: new Date(),
}; };
console.log(title); if ($('.breadcrumb')) {
dataForIndex.breadcrumb = checkData.cleanText($('.breadcrumb').text());
if ($('div').hasClass('.breadcrumb')) {
dataForIndex.breadcrumb = $('.breadcrumb').text();
} }
if ($('h1').length) { if ($('h1').length) {
...@@ -112,7 +105,7 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -112,7 +105,7 @@ export default class IndexWebsite extends IndexGeneric {
index: { index: {
_index: Meteor.settings.private.elasticsearch.esIndex, _index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType, _type: Meteor.settings.private.elasticsearch.esType,
_id: title, _id: url,
}, },
}); });
......
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
import SimpleSchema from 'simpl-schema'; import SimpleSchema from 'simpl-schema';
import { Mongo } from 'meteor/mongo'; import { Mongo } from 'meteor/mongo';
/**
* this local collection keep tracks of all notification of this application
* @type {Mongo.Collection}
*/
const notifsCollection = new Mongo.Collection(null); const notifsCollection = new Mongo.Collection(null);
SimpleSchema.notifsCollection = new SimpleSchema({ SimpleSchema.notifsCollection = new SimpleSchema({
......
import iziToast from 'izitoast'; import iziToast from 'izitoast';
import notifsCollections from '../../../collections/notifsCollection'; import alertsCollections from '../../../collections/alertsCollection';
export default function (options) { export default function (options) {
this.options = { this.options = {
...@@ -14,7 +14,7 @@ export default function (options) { ...@@ -14,7 +14,7 @@ export default function (options) {
Object.assign(this.options, options); Object.assign(this.options, options);
if (options.save === true) { if (options.save === true) {
notifsCollections.insert({ alertsCollections.insert({
type: options.type, type: options.type,
title: options.title, title: options.title,
message: options.message, message: options.message,
......
...@@ -34,10 +34,10 @@ Template.searchTpl.events({ ...@@ -34,10 +34,10 @@ Template.searchTpl.events({
const term = event.target.value; const term = event.target.value;
Meteor.callPromise('autoCompletion', term) /* Meteor.callPromise('autoCompletion', term)
.then((results) => { .then((results) => {
Session.set('autoCompleteResults', results); Session.set('autoCompleteResults', results);
}); });*/
}, },
'click .autocomplete-title': (event) => { 'click .autocomplete-title': (event) => {
......
...@@ -2,12 +2,12 @@ ...@@ -2,12 +2,12 @@
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
import { Session } from 'meteor/session'; import { Session } from 'meteor/session';
import TabsCollection from '../../../../collections/tabsCollection.js'; import TabsCollection from '../../../../collections/tabsCollection';
import '../../../components/tabs/tabs.js'; import '../../../components/tabs/tabs';
import './main/main.js'; import './main/main';
import './site/site.js'; import './site/site';
import './api/api.js'; import './api/api';
import './indexation.html'; import './indexation.html';
......
/* eslint-disable lodash/prefer-lodash-method */
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import $ from 'jquery';
import displayNotif from '../../../../components/notifs/notifs.js'; import displayNotif from '../../../../components/notifs/notifs.js';
import './site.html'; import './site.html';
......
...@@ -37,7 +37,7 @@ const checkData = { ...@@ -37,7 +37,7 @@ const checkData = {
}, },
slugText(str) { slugText(str) {
return str; return slug(str);
}, },
cleanHtml(html) { cleanHtml(html) {
......
...@@ -1698,6 +1698,10 @@ jodid25519@^1.0.0: ...@@ -1698,6 +1698,10 @@ jodid25519@^1.0.0:
dependencies: dependencies:
jsbn "~0.1.0" jsbn "~0.1.0"
jquery@^3.2.1:
version "3.2.1"
resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.2.1.tgz#5c4d9de652af6cd0a770154a631bba12b015c787"
js-tokens@^3.0.0: js-tokens@^3.0.0:
version "3.0.1" version "3.0.1"
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.1.tgz#08e9f132484a2c45a30907e9dc4d5567b7f114d7" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.1.tgz#08e9f132484a2c45a30907e9dc4d5567b7f114d7"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment