Commit 1ef75d2e authored by Nacim Goura's avatar Nacim Goura

global refactoring

parent aec16e72
......@@ -2,7 +2,6 @@
root = true
[*]
indent_style = tab
indent_size = 4
end_of_line = lf
charset = utf-8
......
/node_modules/*
/.meteor/*
\ No newline at end of file
{
"env": {
"meteor": true,
"es6": true,
"shared-node-browser": true
"node": true,
"browser": true,
"mocha": true,
"es6": true
},
"parser": "babel-eslint",
"parserOptions": {
"allowImportExportEverywhere": true
"allowImportExportEverywhere": true,
"ecmaVersion": 6,
"ecmaFeatures": {
"jsx": false,
"experimentalObjectRestSpread": true
}
},
"plugins": [
"meteor"
"meteor",
"promise",
"lodash"
],
"extends": [
"eslint:recommended",
"plugin:meteor/recommended"
"@meteorjs/eslint-config-meteor",
"plugin:meteor/recommended",
"airbnb",
"plugin:lodash/recommended"
],
"settings": {
"import/resolver": "meteor"
},
"rules": {
"import/no-extraneous-dependencies": 0,
"import/extensions": ["off", "never"],
"no-unused-vars": "warn",
"indent": ["error", "tab", { "SwitchCase": 1 }],
"linebreak-style": ["error", "unix"],
"semi": ["error", "always"],
"no-console": "off",
"eqeqeq": ["error", "always"],
"meteor/no-session": [0]
"indent": ["error", 4, { "SwitchCase": 1 }],
"import/extensions": ["off", "never"],
"import/no-extraneous-dependencies": "off",
"no-underscore-dangle": "off",
"class-methods-use-this": "off"
}
}
node_modules/*
.meteor/*
node_modules/
.idea/*
# This file contains information which helps Meteor properly upgrade your
# app when you run 'meteor update'. You should check it into version control
# with your project.
notices-for-0.9.0
notices-for-0.9.1
0.9.4-platform-file
notices-for-facebook-graph-api-2
1.2.0-standard-minifiers-package
1.2.0-meteor-platform-split
1.2.0-cordova-changes
1.2.0-breaking-changes
1.3.0-split-minifiers-package
1.4.0-remove-old-dev-bundle-link
1.4.1-add-shell-server-package
1.4.3-split-account-service-packages
1.5-add-dynamic-import-package
# This file contains a token that is unique to your project.
# Check it into your repository along with the rest of this directory.
# It can be used for purposes such as:
# - ensuring you don't accidentally deploy one app on top of another
# - providing package authors with aggregated statistics
1rkqou51432njx9lloyp
# Meteor packages used by this project, one per line.
# Check this file (and the other files in this directory) into your repository.
#
# 'meteor add' and 'meteor remove' will edit this file for you,
# but you can also edit it by hand.
# base package
meteor-base@1.1.0 # Packages every Meteor app needs to have
mobile-experience@1.0.4 # Packages for a great mobile UX
mongo@1.1.18 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library
# package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode
standard-minifier-js@2.1.0 # JS minifier run for production mode
es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers.
ecmascript@0.8.0 # Enable ECMAScript2015+ syntax in app code
fourseven:scss # Compile scss to css
fortawesome:fontawesome
# package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
# package for test
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
johanbrook:publication-collector # Test a Meteor publication by collecting its output
# package for data
session@1.1.7
aldeed:collection2-core
dburles:collection-helpers
aldeed:autoform
check@1.2.5
# package for user
accounts-base@1.3.0
accounts-password@1.3.6
alanning:roles
practicalmeteor:chai
# other packages
deanius:promise
dynamic-import
server
browser
accounts-base@1.3.0
accounts-password@1.3.6
alanning:roles@1.2.16
aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1
allow-deny@1.0.5
autoupdate@1.3.12
babel-compiler@6.19.2
babel-runtime@1.0.1
base64@1.0.10
binary-heap@1.0.10
blaze@2.3.2
blaze-html-templates@1.1.2
blaze-tools@1.0.10
boilerplate-generator@1.1.0
caching-compiler@1.1.9
caching-html-compiler@1.1.2
callback-hook@1.0.10
check@1.2.5
coffeescript@1.12.3_1
cosmos:browserify@0.10.0
dburles:collection-helpers@1.1.0
ddp@1.2.5
ddp-client@1.3.4
ddp-common@1.2.8
ddp-rate-limiter@1.0.7
ddp-server@1.3.14
deanius:promise@3.1.3
deps@1.0.12
diff-sequence@1.0.7
dynamic-import@0.1.1
ecmascript@0.8.0
ecmascript-runtime@0.4.1
ecmascript-runtime-client@0.4.2
ecmascript-runtime-server@0.4.1
ejson@1.0.13
email@1.2.3
es5-shim@4.6.15
fastclick@1.0.13
fortawesome:fontawesome@4.7.0
fourseven:scss@4.5.0
geojson-utils@1.0.10
hot-code-push@1.0.4
html-tools@1.0.11
htmljs@1.0.11
http@1.2.12
id-map@1.0.9
johanbrook:publication-collector@1.0.7
jquery@1.11.10
kadira:blaze-layout@2.3.0
kadira:flow-router@2.12.1
launch-screen@1.1.1
livedata@1.0.18
localstorage@1.1.0
logging@1.1.17
meteor@1.6.1
meteor-base@1.1.0
minifier-css@1.2.16
minifier-js@2.1.0
minimongo@1.2.1
mobile-experience@1.0.4
mobile-status-bar@1.0.14
modules@0.9.1
modules-runtime@0.8.0
momentjs:moment@2.18.1
mongo@1.1.18
mongo-id@1.0.6
npm-bcrypt@0.9.2
npm-mongo@2.2.24
observe-sequence@1.0.16
ordered-dict@1.0.9
practicalmeteor:chai@2.1.0_1
practicalmeteor:loglevel@1.2.0_2
practicalmeteor:mocha@2.4.5_6
practicalmeteor:mocha-core@1.0.1
practicalmeteor:sinon@1.14.1_2
promise@0.8.9
raix:eventemitter@0.1.3
random@1.0.10
rate-limit@1.0.8
reactive-dict@1.1.9
reactive-var@1.0.11
reload@1.1.11
retry@1.0.9
routepolicy@1.0.12
service-configuration@1.0.11
session@1.1.7
sha@1.0.9
spacebars@1.0.15
spacebars-compiler@1.1.2
srp@1.0.10
standard-minifier-css@1.3.4
standard-minifier-js@2.1.0
templating@1.3.2
templating-compiler@1.3.2
templating-runtime@1.3.2
templating-tools@1.1.2
tmeasday:check-npm-versions@0.3.1
tmeasday:test-reporter-helpers@0.2.1
tracker@1.1.3
ui@1.0.13
underscore@1.0.10
url@1.1.0
webapp@1.3.16
webapp-hashing@1.0.9
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>meteorSearch</title>
</head>
<head>
<title>meteorSearch</title>
</head>
\ No newline at end of file
// Client entry point, imports all client code
import '../imports/startup/client/index.js';
/**
All CSS
*/
@import "../imports/ui/stylesheets/_variables.scss";
@import "../node_modules/bootstrap-sass/assets/stylesheets/_bootstrap.scss";
@import "../imports/ui/stylesheets/variables";
@import "../node_modules/bootstrap-sass/assets/stylesheets/bootstrap";
@import "../imports/ui/stylesheets/bootswatch";
@import "../imports/ui/stylesheets/main";
// Global namespace
@import "../imports/ui/stylesheets/sidebar.scss";
@import "../imports/ui/stylesheets/tabs.scss";
@import "../imports/ui/stylesheets/other.scss";
@import "../imports/ui/stylesheets/not-found";
@import "../imports/ui/stylesheets/sidebar";
@import "../imports/ui/stylesheets/tabs";
@import "../imports/ui/stylesheets/other";
@import "../imports/ui/stylesheets/izitoast";
@import "../imports/ui/stylesheets/sweetalert2";
@import "../imports/ui/stylesheets/autocomplete";
import {Meteor} from 'meteor/meteor';
import indexWebsite from './indexation/indexation.js';
Meteor.methods({
indexWebsite(url) {
let index = new indexWebsite();
return index.initIndexation(url)
.then((resp) => {
console.log(resp);
return resp;
}).catch((error) => {
console.log(error);
});
},
"indexApi"(){
}
});
import IndexGeneric from './indexGeneric';
export default class IndexApi extends IndexGeneric {
}
import { Meteor } from 'meteor/meteor';
import indexationElastic from '../../libs/elasticsearch/elasticsearch';
const esIndex = Meteor.settings.private.elasticsearch.esIndex;
const esType = Meteor.settings.private.elasticsearch.esType;
export default class IndexGeneric {
indexByBulk(data) {
console.log('indexation');
console.log(data);
return indexationElastic.bulk(data);
}
reIndex() {
const body = {
source: {
index: 'idsearch',
},
dest: {
index: 'idsearch2',
},
};
return indexationElastic.reIndex(body);
}
/**
* init analyser and mapping
*/
initElastic() {
const analyser = {
settings: {
analysis: {
filter: {
french_elision: {
type: 'elision',
articles_case: true,
articles: [
'l', 'm', 't', 'qu', 'n', 's',
'j', 'd', 'c', 'jusqu', 'quoiqu',
'lorsqu', 'puisqu',
],
},
french_stop: {
type: 'stop',
stopwords: '_french_',
},
french_stemmer: {
type: 'stemmer',
language: 'french',
},
},
analyzer: {
analyzer_french: {
tokenizer: 'standard',
char_filter: ['html_strip'],
filter: [
'french_elision',
'lowercase',
'french_stop',
'french_stemmer',
],
},
},
},
},
};
const mapping = {
properties: {
tag: {
type: 'keyword',
},
title: {
type: 'text',
analyzer: 'analyzer_french',
},
title_suggest: {
type: 'completion',
analyzer: 'analyzer_french',
max_input_length: 100,
},
description: {
type: 'text',
analyzer: 'analyzer_french',
},
body: {
type: 'text',
analyzer: 'analyzer_french',
},
html: {
type: 'text',
analyzer: 'standard',
},
url: {
type: 'text',
analyzer: 'standard',
},
h1: {
type: 'text',
analyzer: 'analyzer_french',
},
h2: {
type: 'text',
analyzer: 'analyzer_french',
},
breadcrumb: {
type: 'text',
analyzer: 'analyzer_french',
},
createdAt: {
type: 'date',
},
},
};
return indexationElastic.initAnalyzer(esIndex, analyser)
.then(() => indexationElastic.initMapping(esIndex, esType, mapping));
}
/**
* search
* @param term
* @returns {promise}
*/
search(term) {
const params = {};
if (term) {
params.query = {
match: {
title: {
query: term,
slop: 50,
},
},
};
}
return indexationElastic.search(esIndex, esType, params);
}
/**
* autocompletion
* @param term
* @returns {promise}
*/
autoComplete(term) {
let params = {};
if (term) {
params = {
suggest: {
prefix: term,
completion: {
field: 'title_suggest',
size: 25,
},
},
};
}
return indexationElastic.suggest(esIndex, params);
}
}
import { Meteor } from 'meteor/meteor';
import Sitemapper from 'sitemapper';
import Crawler from 'crawler';
import checkData from '../../utils/checkData';
import IndexGeneric from './indexGeneric';
const sitemap = new Sitemapper();
export default class IndexWebsite extends IndexGeneric {
/**
* realize indexation
* @param url
* @returns Promise
*/
async initIndexation(url) {
let urls = [url];
// si c'est un sitemap on récupère ces url
if (checkData.isSitemap(url)) {
const { sites } = await sitemap.fetch(url);
urls = sites;
}
return this.crawlUrl(urls)
.then(dataToIndex => this.indexByBulk(dataToIndex))
.catch((error) => {
throw error;
});
}
/**
* crawl list urls
* @param urls
* @returns {Promise}
*/
crawlUrl(urls) {
return new Promise((resolve, reject) => {
if (urls && urls.length === 0) {
reject(new Meteor.Error('indexation', 'Aucune url fourni!'));
}
this.listDataError = [];
this.listDataForIndex = [];
const crawl = new Crawler({
callback: (error, res, done) => {
if (error) {
this.listDataError.push({
url: res.options.uri,
error,
});
} else {
if (res.statusCode !== 200) {
this.listDataError.push({
url: res.options.uri,
error: res.statusCode,
});
}
this.parseData(res.$, res.options.uri);
}
done();
},
});
crawl.queue(urls);
crawl.on('drain', () => {
console.error(this.listDataError);
resolve(this.listDataForIndex);
});
});
}
/**
* map data
* @param $
* @param url
*/
parseData($, url) {
const body = $('body');
const title = $('title').text();
const dataForIndex = {
tag: 'site',
title,
title_suggest: {
input: title,
},
description: $('meta[name=description]').attr('content'),
body: checkData.cleanText(body.text()),
html: checkData.cleanHtml(body.html()),
url: decodeURI(url),
createdAt: new Date(),
};
console.log(title);
if ($('div').hasClass('.breadcrumb')) {
dataForIndex.menu = $('.breadcrumb').text();
}
if ($('h1').length) {
dataForIndex.h1 = checkData.cleanText($('h1').text());
}
if ($('h2').length) {
dataForIndex.h2 = checkData.cleanText($('h2').text());
}
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType,
_id: title,
},
});
this.listDataForIndex.push(dataForIndex);
}
}
import {Meteor} from 'meteor/meteor';
import Sitemapper from 'sitemapper';
import checkData from '../../utils/checkData.js';
import Crawler from 'crawler';
import indexationElastic from '../../libs/elasticsearch/elasticsearch.js';
export default class indexWebsite {
/**
* init crawl
*/
constructor() {
this.listWebsite = [];
this.listDataForIndex = [];
this.listDataError = [];
}
/**
* init index
* @param url website
*/
async initIndexation(url) {
if(checkData.sitemap(url)) {
return this.getUrlMap(url);
}else if(checkData.url(url)) {
return url;
}else {
throw new Meteor.Error("invalidUrl", "La donnée fournie n'est pas une url valide!");
}
}
/**
*
* @param url sitemap
* @returns all site in a sitemap
*/
async getUrlMap(url) {
const sitemap = new Sitemapper();
return await sitemap.fetch(url);
}
/**
* crawl all url and index at end
* @param url website
*/
crawlUrl(url){
const crawl = new Crawler({
callback : (error, res, done) => {
if(error){
this.listDataError.push({
url: res.options.uri,
error: error
});
}else{
if (res.statusCode !== 200) {
this.listDataError.push({
url: res.options.uri,
error: res.statusCode
});
}else {
this.parseData(res.$, res.options.uri);
}
}
done();
}
});
crawl.queue(url);
crawl.on('drain', () => {
this.indexData();
});
}
/**
* mapping data of website
* @param $ jquery dom
* @param url url website
*/
parseData($, url) {
let body = $('body');
let dataForIndex = {
tag: "site",
title: $('title').text(),
description: $('meta[name=description]').attr('content'),
body: body.text().replace(/\n/g, " ").replace(/function.*}/g, " "),
createdAt: new Date(),
url: decodeURI(url)
};
if($('div').hasClass('.breadcrumb')) {
dataForIndex.menu = $('.breadcrumb').text();