Commit fc0518f2 authored by Nacim Goura's avatar Nacim Goura

global optimization and demo

parent 22f1647e
......@@ -24,7 +24,6 @@ fortawesome:fontawesome # pretty icon
# package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
ajduke:bootstrap-tagsinput # tag input
# package for test or validate
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
......
accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.3
ajduke:bootstrap-tagsinput@0.7.1
alanning:roles@1.2.16
aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1
......@@ -105,7 +104,6 @@ templating-tools@1.1.2
tmeasday:check-npm-versions@0.3.1
tmeasday:test-reporter-helpers@0.2.1
tracker@1.1.3
twbs:bootstrap@3.3.6
ui@1.0.13
underscore@1.0.10
url@1.1.0
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormAccount = {
onSuccess(formType, result) {
displayNotif({
type: 'success',
title: 'Succès : ',
message: `Compte ajouté avec succès pour ${result.username}!`,
save: true,
});
},
onError(formType, result) {
displayNotif({
type: 'error',
title: 'Erreur création de compte : ',
message: result.message,
save: true,
});
},
};
AutoForm.hooks({
formAccount: hooksFormAccount,
});
import formAccountSchema from '/imports/api/account/formAccountSchema';
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import { Accounts } from 'meteor/accounts-base';
import { Roles } from 'meteor/alanning:roles';
import formAccountSchema from '/imports/api/account/formAccountSchema';
/**
* add user account
......@@ -13,8 +14,10 @@ export function addAccount(user) {
formAccountSchema.validate(user);
const id = Accounts.createUser(user);
if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter!');
throw new Meteor.Error('Error', 'Impossible de créer un compte!');
}
Roles.addUsersToRoles(id, 'gestionnaire');
return user;
}
/**
......
......@@ -14,20 +14,46 @@ const configCollection = new Mongo.Collection('configs');
SimpleSchema.configCollection = new SimpleSchema({
userId: {
type: String,
required: false,
autoform: {
type: 'hidden',
},
},
listConfig: {
type: Array,
label: 'Configuration des sites web',
},
'listConfig.$': {
type: Object,
label: 'Configuration',
},
domain: {
'listConfig.$.domain': {
type: String,
label: 'Nom de domaine',
regEx: SimpleSchema.RegEx.Url,
required: false,
autoValue() {
return this.value.replace('https', 'http');
},
},
'listConfig.$.breadcrumb': {
type: String,
label: 'Element du breadcrumb',
},
breadcrumb: {
'listConfig.$.forbiddenWordString': {
type: String,
label: 'Mot non indexable (à séparer par une virgule)',
autoform: {
class: 'forbiddenWordWebsite',
},
},
forbiddenWord: {
'listConfig.$.forbiddenWord': {
type: Array,
required: false,
autoform: {
type: 'hidden',
},
},
'forbiddenWord.$': {
'listConfig.$.forbiddenWord.$': {
type: String,
},
}, { tracker: Tracker });
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormConfig = {
onSuccess(formType, result) {
displayNotif({
type: 'success',
title: 'Configuration : ',
message: 'Configuration ajouté avec succès!',
});
},
onError(formType, error) {
displayNotif({
type: 'error',
title: 'Configuration : ',
message: JSON.stringify(error.message),
save: true,
});
},
};
AutoForm.hooks({
formConfig: hooksFormConfig,
});
import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import _ from 'lodash';
import configCollection from '/imports/api/config/configCollection';
/**
......@@ -8,29 +9,18 @@ import configCollection from '/imports/api/config/configCollection';
* @param config
*/
export function defineConfig(config) {
console.log('define config');
check(config, Object);
configCollection.validate(config);
const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch();
if (config.forbiddenWord) {
config.forbiddenWord = config.forbiddenWord.split(',');
}
if (config.domain) {
config.domain = config.domain.replace('https', 'http');
}
// if config already exist, replace old value with new
if (oldConfig && oldConfig.length) {
configCollection.update(oldConfig[0]._id, {
$set: {
domain: config.domain || oldConfig.domain,
forbiddenWord: config.forbiddenWord || oldConfig.forbiddenWord,
breadcrumb: config.breadcrumb || oldConfig.breadcrumb,
},
});
} else {
// else insert
config.userId = Meteor.userId();
configCollection.insert(config);
if (!config.listConfig) {
throw new Meteor.Error('Error', 'Impossible de définir les configurations');
}
config.userId = Meteor.userId();
_.forEach(config.listConfig, (item) => {
item.forbiddenWord = item.forbiddenWordString.split(',');
});
configCollection.simpleSchema().validate(config);
configCollection.remove({ userId: Meteor.userId() });
configCollection.insert(config);
}
/**
......@@ -42,7 +32,6 @@ export function getConfig() {
}
Meteor.methods({
// define config
defineConfig,
getConfig,
});
import { Meteor } from 'meteor/meteor';
import configCollection from '/imports/api/config/configCollection';
Meteor.publish('config', () => configCollection.find({ userId: Meteor.userId() }));
......@@ -33,7 +33,7 @@ export default class CrawlFacebook {
return new Promise((resolve) => {
_.forEach(this.content.data, (item, index) => {
const dataForIndex = {
tag: 'social',
tag: 'api',
apiName: 'facebook',
domain: this.config.idPage,
url: `https://www.facebook.com/${item.id}`,
......
......@@ -42,7 +42,7 @@ export default class CrawlTwitter {
return new Promise((resolve) => {
_.forEach(tweets, (item, index) => {
const dataForIndex = {
tag: 'social',
tag: 'api',
apiName: 'twitter',
domain: this.config.idPage,
url: `https://twitter.com/${this.config.idPage}/status/${item.id_str}`,
......
......@@ -3,6 +3,7 @@ import { Meteor } from 'meteor/meteor';
import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore';
import Files from '/imports/api/crawl/network/networkCollection';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormNetworkCrawl = {
onSubmit(insertDoc, updateDoc, currentDoc) {
......@@ -32,7 +33,6 @@ const hooksFormNetworkCrawl = {
});
upload.on('end', (error, fileObj) => {
console.log(fileObj.name);
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
......@@ -62,24 +62,9 @@ const hooksFormNetworkCrawl = {
console.log('indexation');
Meteor.callPromise('indexNetwork')
.then((result) => {
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
this.done();
this.done(null, result);
}).catch((error) => {
console.log(error);
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
this.done();
this.done(error);
});
}
};
......@@ -88,6 +73,23 @@ const hooksFormNetworkCrawl = {
}
return false;
},
onSuccess: (formType, result) => {
displayNotif({
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
},
onError: (formType, error) => {
displayNotif({
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
},
};
AutoForm.hooks({
......
......@@ -20,6 +20,10 @@ export default new SimpleSchema({
{ label: 'doc', value: 'doc' },
{ label: 'docx', value: 'docx' },
{ label: 'txt', value: 'txt' },
{ label: 'rtf', value: 'rtf' },
{ label: 'odf', value: 'odf' },
{ label: 'epub', value: 'epub' },
{ label: 'xml', value: 'xml' },
],
},
},
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormWebsiteCrawl = {
onSuccess(formType, result) {
Meteor.call('addNotif', {
displayNotif({
type: 'success',
title: 'Indexation : ',
message: `Indexation fini avec succès avec ${result.index.items.length} urls indexées!`,
message: result.message,
save: true,
});
},
onError(formType, error) {
Meteor.call('addNotif', {
displayNotif({
type: 'error',
title: 'Indexation : ',
message: error.reason ? error.reason : error,
message: JSON.stringify(error.message),
save: true,
});
},
......
......@@ -8,4 +8,8 @@ export default new SimpleSchema({
type: String,
label: 'Url du site ou du sitemap à indexer :',
},
config: {
type: String,
label: 'Configuration à appliquer :',
},
}, { tracker: Tracker });
......@@ -4,24 +4,21 @@ import { Meteor } from 'meteor/meteor';
import Crawler from 'crawler';
import Sitemapper from 'sitemapper';
import checkData from '/imports/utils/checkData';
import { getConfig } from '/imports/api/config/methods';
export default class crawlWebsite {
/**
* crawl list urls
* @param urlWebsite
* @param data
* @returns {Promise}
*/
constructor(urlWebsite) {
this.urlWebsite = urlWebsite;
constructor(data) {
this.urlWebsite = data.urlWebsite;
this.config = {
domain: url.parse(urlWebsite).hostname,
domain: url.parse(this.urlWebsite).hostname,
forbiddenWord: [],
};
if (getConfig()) {
this.config = getConfig();
}
this.config = JSON.parse(data.config);
return this.start();
}
......
......@@ -2,11 +2,12 @@
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import Files from '/imports/api/crawl/network/networkCollection';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import formWebsiteCrawlSchema from '/imports/api/crawl/website/formWebsiteCrawlSchema';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
Meteor.methods({
initIndexElastic() {
......@@ -19,10 +20,11 @@ Meteor.methods({
const index = new IndexGeneric();
return index.reIndex();
},
indexWebsite(url) {
check(url, String);
indexWebsite(data) {
check(data, Object);
formWebsiteCrawlSchema.validate(data);
const index = new IndexWebsite();
return index.start(url);
return index.start(data);
},
indexApi(data) {
check(data, Object);
......
......@@ -6,9 +6,9 @@ export default class IndexNetwork extends IndexGeneric {
async start() {
try {
console.log('start indexation network');
const dataToIndex = await new CrawlNetwork();
const resultIndex = await this.indexByBulk(dataToIndex);
console.log('start indexation network');
const resultIndex = await this.indexByBulk(dataToIndex, true);
return {
message: `Indexation de ${resultIndex.items.length} fichiers fini avec succès!`,
};
......
......@@ -6,20 +6,20 @@ export default class IndexWebsite extends IndexGeneric {
/**
* realize indexation
* @param urlWebsite
* @param data
* @returns Promise
*/
async start(urlWebsite) {
async start(data) {
try {
const dataToIndex = await new CrawlWebsite(urlWebsite);
const dataToIndex = await new CrawlWebsite(data);
/**
* dataToIndex index website page
* false no file
*/
const resultIndex = await this.indexByBulk(dataToIndex, false);
return Promise.resolve({
message: `Indexation de ${resultIndex.items.length} liens pour ${urlWebsite} fini avec succès!`,
});
return {
message: `Indexation de ${resultIndex.items.length} liens pour ${data.urlWebsite} fini avec succès!`,
};
} catch (error) {
throw error;
}
......
import { AutoForm } from 'meteor/aldeed:autoform';
import { Meteor } from 'meteor/meteor';
import { Session } from 'meteor/session';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormSearch = {
onSuccess(formType, result) {
Session.set('websiteResults', result.website);
Session.set('apiResults', result.api);
Session.set('documentResults', result.document);
},
onError(formType, error) {
displayNotif({
type: 'error',
title: 'Recherche : ',
message: JSON.stringify(error.message),
save: true,
});
},
};
......
......@@ -15,8 +15,9 @@ export async function searchWebsite(data) {
total: results.hits.total,
list: _.map(results.hits.hits, '_source'),
};
} catch (err) {
throw new Meteor.Error('Error', err);
} catch (error) {
console.log(error);
throw error;
}
}
......@@ -29,8 +30,9 @@ export async function searchApi(data) {
total: results.hits.total,
list: _.map(results.hits.hits, '_source'),
};
} catch (err) {
throw new Meteor.Error('Error', err);
} catch (error) {
console.log(error);
throw error;
}
}
......@@ -43,17 +45,38 @@ export async function searchDocument(data) {
total: results.hits.total,
list: _.map(results.hits.hits, '_source'),
};
} catch (err) {
throw new Meteor.Error('Error', err);
} catch (error) {
console.log(error);
throw error;
}
}
export async function searchAll(data) {
return {
website: await searchWebsite(data),
api: await searchApi(data),
document: await searchDocument(data),
};
try {
return {
website: await searchWebsite(data),
api: await searchApi(data),
document: await searchDocument(data),
};
} catch (error) {
console.log(error);
throw error;
}
}
export async function autoCompletion(term) {
check(term, String);
const search = new Search();
try {
const results = await search.autoComplete(term);
if (results.suggest) {
return _.take(_.map(results.suggest[0].options, 'text'), 5);
}
return [];
} catch (error) {
console.log(error);
throw error;
}
}
Meteor.methods({
......@@ -61,15 +84,5 @@ Meteor.methods({
searchApi,
searchDocument,
searchAll,
autoCompletion(term) {
check(term, String);
const search = new Search();
return search.autoComplete(term)
.then((result) => {
if (result.suggest) {
return _.take(_.map(result.suggest[0].options, 'text'), 5);
}
return [];
});
},
autoCompletion,
});
......@@ -25,40 +25,96 @@ export default class Search {
*/
params.query = {
bool: {
must: [
{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description.stemmed',
'body.stemmed',
'urlText.stemmed',
'title.stemmed',
],
},
filter: {
term: {
tag: 'site',
},
},
must: [{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description.stemmed',
'body.stemmed',
'urlText.stemmed',
'title.stemmed',
],
},
},
],
should: [
{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description',
'urlText',
'body',
'title',
'url',
'breadcrumb',
'h1',
'h2',
'html',
'url',
],
},
should: [{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description',
'urlText',
'body',
'title',
'url',
'breadcrumb',
'h1',
'h2',
'html',
'url',
],
},
}],
minimum_should_match: 1,
boost: 2.0,
},
};
}
return indexationElastic.search(esIndex, esType, params);
}
searchApi(term) {
const params = {
from: 0,
size: 10,
};
if (term) {
params.query = {
bool: {
filter: {
term: {
tag: 'api',
},
},
must: [{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description.stemmed',
'body.stemmed',
],
},
},
],
should: [{
multi_match: {
query: term,
fuzziness: 'AUTO',
fields: [
'description',
'urlText',
'urlText.stemmed',
'title.stemmed',
'body',
'title',
'url',
'breadcrumb',
'h1',
'h2',
'html',
'url',
],
},
}],
minimum_should_match: 1,
boost: 2.0,
},
......@@ -69,11 +125,52 @@ export default class Search {
}
searchDocument(term) {
return Promise.resolve(term);
}
const params = {
from: 0,
size: 10,
};
searchApi(term) {