Commit aec16e72 authored by Nacim Goura's avatar Nacim Goura

add elasticsearch and promise

parent 15e73cb2
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
"linebreak-style": ["error", "unix"], "linebreak-style": ["error", "unix"],
"semi": ["error", "always"], "semi": ["error", "always"],
"no-console": "off", "no-console": "off",
"eqeqeq": ["error", "always"],
"meteor/no-session": [0] "meteor/no-session": [0]
} }
} }
import '../imports/startup/client'; import '../imports/startup/client/index.js';
import {Meteor} from 'meteor/meteor';
import indexWebsite from './indexation/indexation.js';
Meteor.methods({
indexWebsite(url) {
let index = new indexWebsite();
return index.initIndexation(url)
.then((resp) => {
console.log(resp);
return resp;
}).catch((error) => {
console.log(error);
});
},
"indexApi"(){
}
});
import {Meteor} from 'meteor/meteor';
import Sitemapper from 'sitemapper';
import checkData from '../../utils/checkData.js';
import Crawler from 'crawler';
import indexationElastic from '../../libs/elasticsearch/elasticsearch.js';
export default class indexWebsite {
/**
* init crawl
*/
constructor() {
this.listWebsite = [];
this.listDataForIndex = [];
this.listDataError = [];
}
/**
* init index
* @param url website
*/
async initIndexation(url) {
if(checkData.sitemap(url)) {
return this.getUrlMap(url);
}else if(checkData.url(url)) {
return url;
}else {
throw new Meteor.Error("invalidUrl", "La donnée fournie n'est pas une url valide!");
}
}
/**
*
* @param url sitemap
* @returns all site in a sitemap
*/
async getUrlMap(url) {
const sitemap = new Sitemapper();
return await sitemap.fetch(url);
}
/**
* crawl all url and index at end
* @param url website
*/
crawlUrl(url){
const crawl = new Crawler({
callback : (error, res, done) => {
if(error){
this.listDataError.push({
url: res.options.uri,
error: error
});
}else{
if (res.statusCode !== 200) {
this.listDataError.push({
url: res.options.uri,
error: res.statusCode
});
}else {
this.parseData(res.$, res.options.uri);
}
}
done();
}
});
crawl.queue(url);
crawl.on('drain', () => {
this.indexData();
});
}
/**
* mapping data of website
* @param $ jquery dom
* @param url url website
*/
parseData($, url) {
let body = $('body');
let dataForIndex = {
tag: "site",
title: $('title').text(),
description: $('meta[name=description]').attr('content'),
body: body.text().replace(/\n/g, " ").replace(/function.*}/g, " "),
createdAt: new Date(),
url: decodeURI(url)
};
if($('div').hasClass('.breadcrumb')) {
dataForIndex.menu = $('.breadcrumb').text();
}
if($('h1').length) {
dataForIndex.h1 = $('h1').text();
}
this.listDataForIndex.push({index: {
_index: 'idsearch',
_type: 'haute-savoie',
_id: $('title').text()
}});
this.listDataForIndex.push(dataForIndex);
}
/**
* index all data of website after crawl
*/
indexData() {
const indexation = new indexationElastic();
indexation.bulk(this.listDataForIndex)
.then(() => {
this.endIndexation();
}).catch((error) => {
throw new Meteor.Error("IndexationError", "Une erreur s'est produite lors de l'indexation!", error);
});
}
endIndexation() {
console.error(this.listDataError);
}
}
'use strict';
import elasticSearch from 'elasticsearch';
// Connect to localhost:9200 and use the default settings
const client = new elasticSearch.Client({
host: '127.0.0.1:9201'
});
export default class indexationElastic {
constructor() {
console.log("init index");
}
index(esIndex, esType, id, obj) {
return client.index({
index: esIndex,
type: esType,
id: id,
body: obj
}).then((resp) => {
return resp;
}).catch((error) => {
return error;
});
}
bulk(body) {
return client.bulk({
body: body
}).then((resp) => {
return resp;
}).catch((error) => {
return error;
});
}
}
/**
* Check is index exists or not
*/
exports.indexExists = function (esIndex, callback) {
client.indices.exists({
index: esIndex,
ignore: [404]
}, function (err, exists) {
if (callback) {
callback(err, exists);
}
});
};
/**
* Close index
*/
exports.indexClose = function (esIndex, callback) {
client.indices.close({
index: esIndex,
ignore: [404]
}, function (err, resp) {
if (callback) {
callback(err, resp);
}
});
};
/**
* Open index
*/
exports.indexOpen = function (esIndex, callback) {
client.indices.open({
index: esIndex,
ignore: [404]
}, function (err, exists) {
if (callback) {
callback(err, exists);
}
});
};
/**
* Create index
*/
exports.indexCreate = function (esIndex, obj, callback) {
client.indices.create({
index: esIndex,
body: obj
}, function (err, resp) {
if (callback) {
callback(err, resp);
}
});
};
/**
* Delete index
*/
exports.indexDelete = function (esIndex, callback) {
client.indices.delete({
index: esIndex,
ignore: [404]
}, function (err, resp) {
if (callback) {
callback(err, resp);
}
});
};
/**
* Set index setting
*/
exports.indexSetSettings = function (esIndex, obj, callback) {
client.indices.putSettings({
index: esIndex,
body: obj
}, function (err, resp) {
if (callback) {
callback(err, resp);
}
});
};
/**
* Check is type exists or not
*/
exports.typeExists = function (esIndex, esType, callback) {
client.indices.existsType({
index: esIndex,
type: esType
}, function (err, exists) {
if (callback) {
callback(err, exists);
}
});
};
/**
* Create mapping
*/
exports.mappingCreate = function (esIndex, esType, obj, callback) {
client.indices.putMapping({
index: esIndex,
type: esType,
body: obj
}, function (err, resp) {
if (callback) {
callback(err, resp);
}
});
};
/**
* Delete mapping
*/
exports.mappingDelete = function (esIndex, esType, callback) {
this.indexClose(esIndex, function (err) {
if (err) {
console.log('Error in elasticsearch mappingDelete() : indexClose() ' + err);
}
client.indices.deleteMapping({
index: esIndex,
type: esType
}, function (err, resp) {
if (err) {
console.log('Error in elasticsearch mappingDelete() : indexClose() ' + err);
}
if (callback) {
callback(err, resp);
}
});
});
};
/**
* Index object
*/
exports.index = function (esIndex, esType, id, obj, callback) {
client.index({
index: esIndex,
type: esType,
id: id,
body: obj,
requestTimeout: 120000
}, function (err, resp) {
if (err) {
console.log('Error in elasticsearch index() : ' + err);
}
if (callback) {
callback(err, resp);
}
});
};
/**
* Update object
*/
// exports.update = function (esIndex, esType, id, obj, callback) {
// client.update({
// index: esIndex,
// type: esType,
// id: id,
// body: obj
// }, function (err, resp) {
// if (err) {
// console.log('Error in elasticsearch update() : ' + err);
// }
//
// if (callback) {
// callback(err, resp);
// }
// });
// };
/**
* Delete object
*/
exports.delete = function (esIndex, esType, id, callback) {
client.delete({
index: esIndex,
type: esType,
id: id
}, function (err, resp) {
if (err) {
//console.log('Error in elasticsearch delete() : ' + err, id);
}
if (callback) {
callback(err, resp);
}
});
};
/**
* Bulk
*/
exports.bulk = function (body, callback) {
client.bulk({
body: body
}, function (err, resp) {
if (err) {
console.log('Error in elasticsearch bulk() : ' + err);
}
if (callback) {
callback(err, resp);
}
// client.indices.clearCache({index:["pple"]}, function(errorFullMessage, display, status) {
// if (errorFullMessage) {
// console.log('clear-cache', display.status, display.error.reason, '"' + display.error.index + '"');
// // } else {
// // console.log(display);
// }
// if (callback) {
// callback(err, resp);
// }
// });
});
};
/**
* Search object
*/
exports.search = function (esIndex, esType, params, callback) {
client.search({
index: esIndex,
type: esType,
body: params
}, function (err, exists) {
if (callback) {
callback(err, exists);
}
});
};
/**
* Init analyser
*/
exports.initAnalyser = function (esIndex, obj, callback) {
var _this = this;
this.indexExists(esIndex, function (err, exists) {
if (err) {
console.log('Error in elasticsearch initAnalyser() - indexExists() : ' + err);
}
if (exists) {
_this.indexDelete(esIndex, function (err) {
if (err) {
console.log('Error in elasticsearch initAnalyser() : indexDelete() ' + err);
}
_this.indexCreate(esIndex, obj, callback);
});
}
else {
_this.indexCreate(esIndex, obj, callback);
}
});
};
/**
* Init analyser
*/
exports.initMapping = function (esIndex, esType, obj, callback) {
var _this = this;
console.log('===================================================');
console.log('esIndex',esIndex);
console.log('esType',esType);
this.typeExists(esIndex, esType, function (err, exists) {
if (err) {
console.log('Error in elasticsearch initMapping() - typeExists() : ' + err);
}
console.log('exists', exists);
console.log('===================================================');
if (exists) {
_this.mappingDelete(esIndex, esType, function (err) {
if (err) {
console.log('Error in elasticsearch initMapping() : mappingDelete() ' + err);
}
_this.mappingCreate(esIndex, esType, obj, function (err) {
if (err) {
console.log('Error in elasticsearch initMapping() : mappingCreate() ' + err);
}
_this.indexOpen(esIndex, callback);
});
});
}
else {
_this.indexClose(esIndex, function (err) {
if (err) {
console.log('Error in elasticsearch initMapping() : indexClose() ' + err);
}
_this.mappingCreate(esIndex, esType, obj, function (err) {
if (err) {
console.log('Error in elasticsearch initMapping() : mappingCreate() ' + err);
}
_this.indexOpen(esIndex, callback);
});
});
}
});
};
import '../../api/api'; import '../../api/api.js';
...@@ -57,7 +57,7 @@ Template.tabsTpl.onCreated(function () { ...@@ -57,7 +57,7 @@ Template.tabsTpl.onCreated(function () {
// set a cursor observer on tabCollection query // set a cursor observer on tabCollection query
let query = TabsCollection.find({module: module, activ: true}); let query = TabsCollection.find({module: module, activ: true});
let handle = query.observeChanges({ query.observeChanges({
added(id, tab) { added(id, tab) {
// Deactivate previous tabs of the module // Deactivate previous tabs of the module
if (Session.get(module)) { if (Session.get(module)) {
......
import { FlowRouter } from 'meteor/kadira:flow-router';
import './sidebar.html'; import './sidebar.html';
<template name="siteIndexationTpl"> <template name="siteIndexationTpl">
<div class="panel panel-default wrapper"> <div class="panel panel-default wrapper">
<div class="panel-body"> <div class="panel-body">
<form class="form-horizontal"> <div class="alert message"></div>
<form class="form-horizontal" id="submitUrlSite">
<label class="control-label">Url du sitemap (au format xml) ou du site à indexer : </label>
<div class="input-group input-group-lg"> <div class="input-group input-group-lg">
<input type="url" class="form-control" placeholder="Url"> <input type="url" class="form-control" placeholder="Url" required="required" name="urlSite">
<span class="input-group-addon">Valider</span> <span class="input-group-btn">
<button type="submit" class="btn btn-success">Valider</button>
</span>
</div> </div>
</form> </form>
</div> </div>
......
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
import { Meteor } from 'meteor/meteor';
import './site.html'; import './site.html';
Template.siteIndexationTpl.events({ Template.siteIndexationTpl.events({
'submit form'(event) { 'submit #submitUrlSite'(event) {
event.preventDefault();
const url = event.target.urlSite.value;
$('.message').html("Indexation en cours, vous serez averti à la fin de l'indexation...").removeClass().addClass('alert message alert-info');
Meteor.callPromise('indexWebsite', url)
.then((res) => {
console.log(res);
}).catch((err) => {
$('.message').html(err.reason).removeClass().addClass('alert message alert-danger');
});
event.target.reset();
} }
}); });
<template name="loginTpl"> <template name="loginTpl">
<div class="row">
<!-- template loginTpl -->
<div class="row">
<div class="col-md-6 col-md-offset-3"> <div class="col-md-6 col-md-offset-3">
<form class="signin-form"> <form class="signin-form">
<div class="panel panel-primary"> <div class="panel panel-primary">
...@@ -30,4 +33,4 @@ ...@@ -30,4 +33,4 @@
</form> </form>
</div> </div>
</div> </div>
</template> </template>
\ No newline at end of file
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import { FlowRouter } from 'meteor/kadira:flow-router'; import { FlowRouter } from 'meteor/kadira:flow-router';
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
...@@ -11,6 +12,9 @@ Template.loginTpl.events({ ...@@ -11,6 +12,9 @@ Template.loginTpl.events({
const login = event.target.login.value; const login = event.target.login.value;
const password = event.target.password.value; const password = event.target.password.value;
check(login, String);
check(password, String);
Meteor.loginWithPassword(login, password, (err) => { Meteor.loginWithPassword(login, password, (err) => {
if (err) { if (err) {
$('.message').html(err.reason).addClass('alert-danger'); $('.message').html(err.reason).addClass('alert-danger');
......
import {SimpleSchema} from './definitionSimpleSchema.js';
import { Match } from 'meteor/check';
const checkData = {
/**
* test if string is a url
* @param str string check
* @returns {boolean}
*/
url(str) {
const matchUrl = Match.Where(function(str){
const regexp = new RegExp(SimpleSchema.RegEx.Url);
return regexp.test(str);
});
return Match.test(str, matchUrl);
},
/**
* test if string is a url sitemap
* @param str string check
* @returns {boolean}
*/
sitemap(str) {
return !!this.url(str) && str.split('.').pop() === "xml";
}
};
export default checkData;
import {SimpleSchema} from 'meteor/aldeed:simple-schema';
SimpleSchema.RegEx.Url2 = "https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)";
export {SimpleSchema as SimpleSchema};
...@@ -10,7 +10,10 @@ ...@@ -10,7 +10,10 @@
"babel-runtime": "^6.20.0", "babel-runtime": "^6.20.0",
"bcrypt": "^1.0.2", "bcrypt": "^1.0.2",
"bootstrap-sass": "^3.3.7", "bootstrap-sass": "^3.3.7",
"meteor-node-stubs": "~0.2.4" "crawler": "^1.0.3",
"elasticsearch": "^12.1.3",
"meteor-node-stubs": "~0.2.4",
"sitemapper": "^2.1.7"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^7.1.1", "babel-eslint": "^7.1.1",
......
...@@ -2,6 +2,8 @@ import { Meteor } from 'meteor/meteor'; ...@@ -2,6 +2,8 @@ import { Meteor } from 'meteor/meteor';
import { Accounts } from 'meteor/accounts-base'; import { Accounts } from 'meteor/accounts-base';
import { Roles} from 'meteor/alanning:roles'; import { Roles} from 'meteor/alanning:roles';
import '../imports/startup/server/index.js';
Meteor.startup(() => { Meteor.startup(() => {
/** /**
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment