Commit fc0518f2 authored by Nacim Goura's avatar Nacim Goura

global optimization and demo

parent 22f1647e
...@@ -24,7 +24,6 @@ fortawesome:fontawesome # pretty icon ...@@ -24,7 +24,6 @@ fortawesome:fontawesome # pretty icon
# package for view # package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter) kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
ajduke:bootstrap-tagsinput # tag input
# package for test or validate # package for test or validate
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
......
accounts-base@1.3.1-rc.3 accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.3 accounts-password@1.4.0-rc.3
ajduke:bootstrap-tagsinput@0.7.1
alanning:roles@1.2.16 alanning:roles@1.2.16
aldeed:autoform@6.2.0 aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1 aldeed:collection2-core@2.0.1
...@@ -105,7 +104,6 @@ templating-tools@1.1.2 ...@@ -105,7 +104,6 @@ templating-tools@1.1.2
tmeasday:check-npm-versions@0.3.1 tmeasday:check-npm-versions@0.3.1
tmeasday:test-reporter-helpers@0.2.1 tmeasday:test-reporter-helpers@0.2.1
tracker@1.1.3 tracker@1.1.3
twbs:bootstrap@3.3.6
ui@1.0.13 ui@1.0.13
underscore@1.0.10 underscore@1.0.10
url@1.1.0 url@1.1.0
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormAccount = {
onSuccess(formType, result) {
displayNotif({
type: 'success',
title: 'Succès : ',
message: `Compte ajouté avec succès pour ${result.username}!`,
save: true,
});
},
onError(formType, result) {
displayNotif({
type: 'error',
title: 'Erreur création de compte : ',
message: result.message,
save: true,
});
},
};
AutoForm.hooks({
formAccount: hooksFormAccount,
});
import formAccountSchema from '/imports/api/account/formAccountSchema';
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import { Accounts } from 'meteor/accounts-base'; import { Accounts } from 'meteor/accounts-base';
import { Roles } from 'meteor/alanning:roles';
import formAccountSchema from '/imports/api/account/formAccountSchema';
/** /**
* add user account * add user account
...@@ -13,8 +14,10 @@ export function addAccount(user) { ...@@ -13,8 +14,10 @@ export function addAccount(user) {
formAccountSchema.validate(user); formAccountSchema.validate(user);
const id = Accounts.createUser(user); const id = Accounts.createUser(user);
if (!id) { if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter!'); throw new Meteor.Error('Error', 'Impossible de créer un compte!');
} }
Roles.addUsersToRoles(id, 'gestionnaire');
return user;
} }
/** /**
......
...@@ -14,20 +14,46 @@ const configCollection = new Mongo.Collection('configs'); ...@@ -14,20 +14,46 @@ const configCollection = new Mongo.Collection('configs');
SimpleSchema.configCollection = new SimpleSchema({ SimpleSchema.configCollection = new SimpleSchema({
userId: { userId: {
type: String, type: String,
required: false,
autoform: {
type: 'hidden',
},
},
listConfig: {
type: Array,
label: 'Configuration des sites web',
},
'listConfig.$': {
type: Object,
label: 'Configuration',
}, },
domain: { 'listConfig.$.domain': {
type: String, type: String,
label: 'Nom de domaine',
regEx: SimpleSchema.RegEx.Url, regEx: SimpleSchema.RegEx.Url,
required: false, autoValue() {
return this.value.replace('https', 'http');
},
},
'listConfig.$.breadcrumb': {
type: String,
label: 'Element du breadcrumb',
}, },
breadcrumb: { 'listConfig.$.forbiddenWordString': {
type: String, type: String,
label: 'Mot non indexable (à séparer par une virgule)',
autoform: {
class: 'forbiddenWordWebsite',
},
}, },
forbiddenWord: { 'listConfig.$.forbiddenWord': {
type: Array, type: Array,
required: false, required: false,
autoform: {
type: 'hidden',
},
}, },
'forbiddenWord.$': { 'listConfig.$.forbiddenWord.$': {
type: String, type: String,
}, },
}, { tracker: Tracker }); }, { tracker: Tracker });
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormConfig = {
onSuccess(formType, result) {
displayNotif({
type: 'success',
title: 'Configuration : ',
message: 'Configuration ajouté avec succès!',
});
},
onError(formType, error) {
displayNotif({
type: 'error',
title: 'Configuration : ',
message: JSON.stringify(error.message),
save: true,
});
},
};
AutoForm.hooks({
formConfig: hooksFormConfig,
});
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check'; import { check } from 'meteor/check';
import _ from 'lodash';
import configCollection from '/imports/api/config/configCollection'; import configCollection from '/imports/api/config/configCollection';
/** /**
...@@ -8,29 +9,18 @@ import configCollection from '/imports/api/config/configCollection'; ...@@ -8,29 +9,18 @@ import configCollection from '/imports/api/config/configCollection';
* @param config * @param config
*/ */
export function defineConfig(config) { export function defineConfig(config) {
console.log('define config');
check(config, Object); check(config, Object);
configCollection.validate(config); if (!config.listConfig) {
const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch(); throw new Meteor.Error('Error', 'Impossible de définir les configurations');
if (config.forbiddenWord) {
config.forbiddenWord = config.forbiddenWord.split(',');
}
if (config.domain) {
config.domain = config.domain.replace('https', 'http');
}
// if config already exist, replace old value with new
if (oldConfig && oldConfig.length) {
configCollection.update(oldConfig[0]._id, {
$set: {
domain: config.domain || oldConfig.domain,
forbiddenWord: config.forbiddenWord || oldConfig.forbiddenWord,
breadcrumb: config.breadcrumb || oldConfig.breadcrumb,
},
});
} else {
// else insert
config.userId = Meteor.userId();
configCollection.insert(config);
} }
config.userId = Meteor.userId();
_.forEach(config.listConfig, (item) => {
item.forbiddenWord = item.forbiddenWordString.split(',');
});
configCollection.simpleSchema().validate(config);
configCollection.remove({ userId: Meteor.userId() });
configCollection.insert(config);
} }
/** /**
...@@ -42,7 +32,6 @@ export function getConfig() { ...@@ -42,7 +32,6 @@ export function getConfig() {
} }
Meteor.methods({ Meteor.methods({
// define config
defineConfig, defineConfig,
getConfig, getConfig,
}); });
import { Meteor } from 'meteor/meteor';
import configCollection from '/imports/api/config/configCollection';
Meteor.publish('config', () => configCollection.find({ userId: Meteor.userId() }));
...@@ -33,7 +33,7 @@ export default class CrawlFacebook { ...@@ -33,7 +33,7 @@ export default class CrawlFacebook {
return new Promise((resolve) => { return new Promise((resolve) => {
_.forEach(this.content.data, (item, index) => { _.forEach(this.content.data, (item, index) => {
const dataForIndex = { const dataForIndex = {
tag: 'social', tag: 'api',
apiName: 'facebook', apiName: 'facebook',
domain: this.config.idPage, domain: this.config.idPage,
url: `https://www.facebook.com/${item.id}`, url: `https://www.facebook.com/${item.id}`,
......
...@@ -42,7 +42,7 @@ export default class CrawlTwitter { ...@@ -42,7 +42,7 @@ export default class CrawlTwitter {
return new Promise((resolve) => { return new Promise((resolve) => {
_.forEach(tweets, (item, index) => { _.forEach(tweets, (item, index) => {
const dataForIndex = { const dataForIndex = {
tag: 'social', tag: 'api',
apiName: 'twitter', apiName: 'twitter',
domain: this.config.idPage, domain: this.config.idPage,
url: `https://twitter.com/${this.config.idPage}/status/${item.id_str}`, url: `https://twitter.com/${this.config.idPage}/status/${item.id_str}`,
......
...@@ -3,6 +3,7 @@ import { Meteor } from 'meteor/meteor'; ...@@ -3,6 +3,7 @@ import { Meteor } from 'meteor/meteor';
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore'; import { _ } from 'meteor/underscore';
import Files from '/imports/api/crawl/network/networkCollection'; import Files from '/imports/api/crawl/network/networkCollection';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormNetworkCrawl = { const hooksFormNetworkCrawl = {
onSubmit(insertDoc, updateDoc, currentDoc) { onSubmit(insertDoc, updateDoc, currentDoc) {
...@@ -32,7 +33,6 @@ const hooksFormNetworkCrawl = { ...@@ -32,7 +33,6 @@ const hooksFormNetworkCrawl = {
}); });
upload.on('end', (error, fileObj) => { upload.on('end', (error, fileObj) => {
console.log(fileObj.name);
if (error) { if (error) {
listAlertNetwork.push({ listAlertNetwork.push({
type: 'alert-danger', type: 'alert-danger',
...@@ -62,24 +62,9 @@ const hooksFormNetworkCrawl = { ...@@ -62,24 +62,9 @@ const hooksFormNetworkCrawl = {
console.log('indexation'); console.log('indexation');
Meteor.callPromise('indexNetwork') Meteor.callPromise('indexNetwork')
.then((result) => { .then((result) => {
Meteor.call('addNotif', { this.done(null, result);
userId: Meteor.userId(),
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
this.done();
}).catch((error) => { }).catch((error) => {
console.log(error); this.done(error);
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
this.done();
}); });
} }
}; };
...@@ -88,6 +73,23 @@ const hooksFormNetworkCrawl = { ...@@ -88,6 +73,23 @@ const hooksFormNetworkCrawl = {
} }
return false; return false;
}, },
onSuccess: (formType, result) => {
displayNotif({
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
},
onError: (formType, error) => {
displayNotif({
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
},
}; };
AutoForm.hooks({ AutoForm.hooks({
......
...@@ -20,6 +20,10 @@ export default new SimpleSchema({ ...@@ -20,6 +20,10 @@ export default new SimpleSchema({
{ label: 'doc', value: 'doc' }, { label: 'doc', value: 'doc' },
{ label: 'docx', value: 'docx' }, { label: 'docx', value: 'docx' },
{ label: 'txt', value: 'txt' }, { label: 'txt', value: 'txt' },
{ label: 'rtf', value: 'rtf' },
{ label: 'odf', value: 'odf' },
{ label: 'epub', value: 'epub' },
{ label: 'xml', value: 'xml' },
], ],
}, },
}, },
......
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormWebsiteCrawl = { const hooksFormWebsiteCrawl = {
onSuccess(formType, result) { onSuccess(formType, result) {
Meteor.call('addNotif', { displayNotif({
type: 'success', type: 'success',
title: 'Indexation : ', title: 'Indexation : ',
message: `Indexation fini avec succès avec ${result.index.items.length} urls indexées!`, message: result.message,
save: true, save: true,
}); });
}, },
onError(formType, error) { onError(formType, error) {
Meteor.call('addNotif', { displayNotif({
type: 'error', type: 'error',
title: 'Indexation : ', title: 'Indexation : ',
message: error.reason ? error.reason : error, message: JSON.stringify(error.message),
save: true, save: true,
}); });
}, },
......
...@@ -8,4 +8,8 @@ export default new SimpleSchema({ ...@@ -8,4 +8,8 @@ export default new SimpleSchema({
type: String, type: String,
label: 'Url du site ou du sitemap à indexer :', label: 'Url du site ou du sitemap à indexer :',
}, },
config: {
type: String,
label: 'Configuration à appliquer :',
},
}, { tracker: Tracker }); }, { tracker: Tracker });
...@@ -4,24 +4,21 @@ import { Meteor } from 'meteor/meteor'; ...@@ -4,24 +4,21 @@ import { Meteor } from 'meteor/meteor';
import Crawler from 'crawler'; import Crawler from 'crawler';
import Sitemapper from 'sitemapper'; import Sitemapper from 'sitemapper';
import checkData from '/imports/utils/checkData'; import checkData from '/imports/utils/checkData';
import { getConfig } from '/imports/api/config/methods';
export default class crawlWebsite { export default class crawlWebsite {
/** /**
* crawl list urls * crawl list urls
* @param urlWebsite * @param data
* @returns {Promise} * @returns {Promise}
*/ */
constructor(urlWebsite) { constructor(data) {
this.urlWebsite = urlWebsite; this.urlWebsite = data.urlWebsite;
this.config = { this.config = {
domain: url.parse(urlWebsite).hostname, domain: url.parse(this.urlWebsite).hostname,
forbiddenWord: [], forbiddenWord: [],
}; };
if (getConfig()) { this.config = JSON.parse(data.config);
this.config = getConfig();
}
return this.start(); return this.start();
} }
......
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
import { check } from 'meteor/check'; import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import Files from '/imports/api/crawl/network/networkCollection'; import Files from '/imports/api/crawl/network/networkCollection';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric'; import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite'; import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi'; import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexNetwork from '/imports/api/indexation/server/indexNetwork'; import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import formWebsiteCrawlSchema from '/imports/api/crawl/website/formWebsiteCrawlSchema';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
Meteor.methods({ Meteor.methods({
initIndexElastic() { initIndexElastic() {
...@@ -19,10 +20,11 @@ Meteor.methods({ ...@@ -19,10 +20,11 @@ Meteor.methods({
const index = new IndexGeneric(); const index = new IndexGeneric();
return index.reIndex(); return index.reIndex();
}, },
indexWebsite(url) { indexWebsite(data) {
check(url, String); check(data, Object);
formWebsiteCrawlSchema.validate(data);
const index = new IndexWebsite(); const index = new IndexWebsite();
return index.start(url); return index.start(data);
}, },
indexApi(data) { indexApi(data) {
check(data, Object); check(data, Object);
......
...@@ -6,9 +6,9 @@ export default class IndexNetwork extends IndexGeneric { ...@@ -6,9 +6,9 @@ export default class IndexNetwork extends IndexGeneric {
async start() { async start() {
try { try {
console.log('start indexation network');
const dataToIndex = await new CrawlNetwork(); const dataToIndex = await new CrawlNetwork();
const resultIndex = await this.indexByBulk(dataToIndex); console.log('start indexation network');
const resultIndex = await this.indexByBulk(dataToIndex, true);
return { return {
message: `Indexation de ${resultIndex.items.length} fichiers fini avec succès!`, message: `Indexation de ${resultIndex.items.length} fichiers fini avec succès!`,
}; };
......
...@@ -6,20 +6,20 @@ export default class IndexWebsite extends IndexGeneric { ...@@ -6,20 +6,20 @@ export default class IndexWebsite extends IndexGeneric {
/** /**
* realize indexation * realize indexation
* @param urlWebsite * @param data
* @returns Promise * @returns Promise
*/ */
async start(urlWebsite) { async start(data) {
try { try {
const dataToIndex = await new CrawlWebsite(urlWebsite); const dataToIndex = await new CrawlWebsite(data);
/** /**
* dataToIndex index website page * dataToIndex index website page
* false no file * false no file
*/ */
const resultIndex = await this.indexByBulk(dataToIndex, false); const resultIndex = await this.indexByBulk(dataToIndex, false);
return Promise.resolve({ return {
message: `Indexation de ${resultIndex.items.length} liens pour ${urlWebsite} fini avec succès!`, message: `Indexation de ${resultIndex.items.length} liens pour ${data.urlWebsite} fini avec succès!`,
}); };
} catch (error) { } catch (error) {
throw error; throw error;
} }
......
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import { Meteor } from 'meteor/meteor';
import { Session } from 'meteor/session'; import { Session } from 'meteor/session';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormSearch = { const hooksFormSearch = {
onSuccess(formType, result) { onSuccess(formType, result) {
Session.set('websiteResults', result.website);
Session.set('apiResults', result.api);
Session.set('documentResults', result.document);
},
onError(formType, error) {
displayNotif({
type: 'error',
title: 'Recherche : ',
message: JSON.stringify(error.message),
save: true,
});
}, },
}; };
......
...@@ -15,8 +15,9 @@ export async function searchWebsite(data) { ...@@ -15,8 +15,9 @@ export async function searchWebsite(data) {
total: results.hits.total, total: results.hits.total,
list: _.map(results.hits.hits, '_source'), list: _.map(results.hits.hits, '_source'),
}; };
} catch (err) { } catch (error) {
throw new Meteor.Error('Error', err); console.log(error);
throw error;
} }
} }
...@@ -29,8 +30,9 @@ export async function searchApi(data) { ...@@ -29,8 +30,9 @@ export async function searchApi(data) {
total: results.hits.total, total: results.hits.total,
list: _.map(results.hits.hits, '_source'), list: _.map(results.hits.hits, '_source'),
}; };
} catch (err) { } catch (error) {
throw new Meteor.Error('Error', err); console.log(error);
throw error;
} }
} }
...@@ -43,17 +45,38 @@ export async function searchDocument(data) { ...@@ -43,17 +45,38 @@ export async function searchDocument(data) {
total: results.hits.total, total: results.hits.total,
list: _.map(results.hits.hits, '_source'), list: _.map(results.hits.hits, '_source'),
}; };
} catch (err) { } catch (error) {
throw new Meteor.Error('Error', err); console.log(error);
throw error;
} }
} }
export async function searchAll(data) { export async function searchAll(data) {
return { try {
website: await searchWebsite(data), return {
api: await searchApi(data), website: await searchWebsite(data),
document: await searchDocument(data), api: await searchApi(data),
}; document: await searchDocument(data),
};
} catch (error) {
console.log(error);
throw error;
}
}
export async function autoCompletion(term) {
check(term, String);
const search = new Search();
try {
const results = await search.autoComplete(term);
if (results.suggest) {
return _.take(_.map(results.suggest[0].options, 'text'), 5);
}
return [];