Commit 6a4282a8 authored by Nacim Goura's avatar Nacim Goura

continue integration job

parent cd174f25
......@@ -7,7 +7,7 @@
# base package
meteor-base@1.1.0 # Packages every Meteor app needs to have
mobile-experience@1.0.4 # Packages for a great mobile UX
mongo@1.1.19-rc.4 # The database Meteor supports right now
mongo@1.1.19 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library
......@@ -15,15 +15,17 @@ dynamic-import@0.1.1
# package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode
standard-minifier-js@2.1.1-rc.4 # JS minifier run for production mode
standard-minifier-js@2.1.1 # JS minifier run for production mode
es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers.
ecmascript@0.8.1 # Enable ECMAScript2015+ syntax in app code
fourseven:scss@4.5.0 # Compile scss to css
fortawesome:fontawesome # pretty icon
# package for view
# package for view and template
kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
aldeed:tabular # table gestion
aldeed:template-extension # add more power for template
# package for test or validate
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
......@@ -39,13 +41,14 @@ aldeed:autoform
matb33:collection-hooks
# package for user
accounts-base@1.3.1-rc.4
accounts-password@1.4.0-rc.4
accounts-base@1.3.1
accounts-password@1.4.0
alanning:roles
# other packages
deanius:promise
aldeed:tabular
percolate:synced-cron
ostrio:files
aldeed:template-extension
tsega:bootstrap3-datetimepicker@=3.1.3_3
aldeed:autoform-bs-datetimepicker
METEOR@1.5.1-rc.4
METEOR@1.5.1
accounts-base@1.3.1-rc.4
accounts-password@1.4.0-rc.4
accounts-base@1.3.1
accounts-password@1.4.0
alanning:roles@1.2.16
aldeed:autoform@6.2.0
aldeed:autoform-bs-datetimepicker@1.0.7
aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1
aldeed:template-extension@4.1.0
allow-deny@1.0.6-rc.4
allow-deny@1.0.6
autoupdate@1.3.12
babel-compiler@6.19.4-rc.4
babel-compiler@6.19.4
babel-runtime@1.0.1
base64@1.0.10
binary-heap@1.0.10
......@@ -22,18 +23,18 @@ check@1.2.5
coffeescript@1.12.6_1
cosmos:browserify@0.10.0
dburles:collection-helpers@1.1.0
ddp@1.3.0-rc.4
ddp-client@2.0.0-rc.4
ddp-common@1.2.9-rc.4
ddp@1.3.0
ddp-client@2.0.0
ddp-common@1.2.9
ddp-rate-limiter@1.0.7
ddp-server@2.0.0-rc.4
ddp-server@2.0.0
deanius:promise@3.1.3
deps@1.0.12
diff-sequence@1.0.7
dynamic-import@0.1.1
ecmascript@0.8.1
ecmascript@0.8.2
ecmascript-runtime@0.4.1
ecmascript-runtime-client@0.4.2
ecmascript-runtime-client@0.4.3
ecmascript-runtime-server@0.4.1
ejson@1.0.13
email@1.2.3
......@@ -53,20 +54,20 @@ kadira:blaze-layout@2.3.0
kadira:flow-router@2.12.1
launch-screen@1.1.1
livedata@1.0.18
localstorage@1.1.0
localstorage@1.1.1
logging@1.1.17
matb33:collection-hooks@0.8.4
meteor@1.7.0-rc.4
meteor@1.7.0
meteor-base@1.1.0
minifier-css@1.2.16
minifier-js@2.1.1-rc.4
minifier-js@2.1.1
minimongo@1.2.1
mobile-experience@1.0.4
mobile-status-bar@1.0.14
modules@0.9.2
modules-runtime@0.8.0
momentjs:moment@2.18.1
mongo@1.1.19-rc.4
mongo@1.1.19
mongo-id@1.0.6
npm-bcrypt@0.9.3
npm-mongo@2.2.24
......@@ -96,7 +97,7 @@ spacebars@1.0.15
spacebars-compiler@1.1.2
srp@1.0.10
standard-minifier-css@1.3.4
standard-minifier-js@2.1.1-rc.4
standard-minifier-js@2.1.1
templating@1.3.2
templating-compiler@1.3.2
templating-runtime@1.3.2
......@@ -104,8 +105,9 @@ templating-tools@1.1.2
tmeasday:check-npm-versions@0.3.1
tmeasday:test-reporter-helpers@0.2.1
tracker@1.1.3
tsega:bootstrap3-datetimepicker@3.1.3_3
ui@1.0.13
underscore@1.0.10
url@1.1.0
webapp@1.3.17-rc.4
webapp@1.3.17
webapp-hashing@1.0.9
......@@ -18,11 +18,7 @@ export function addAccount(user) {
throw new Meteor.Error('Error', 'Impossible de créer un compte!');
}
Roles.addUsersToRoles(id, 'gestionnaire');
console.log(user);
defineConfig({
userId: id,
listConfig: [],
});
defineConfig({ userId: id });
return user;
}
......
......@@ -18,6 +18,9 @@ SimpleSchema.configCollection = new SimpleSchema({
autoform: {
type: 'hidden',
},
autoValue() {
return this.value && this.value.toLowerCase();
},
},
listConfig: {
type: Array,
......@@ -26,6 +29,7 @@ SimpleSchema.configCollection = new SimpleSchema({
'listConfig.$': {
type: Object,
label: 'Configuration',
required: false,
},
'listConfig.$.domain': {
type: String,
......
......@@ -10,20 +10,29 @@ import configCollection from '/imports/api/config/configCollection';
*/
export function defineConfig(config) {
console.log('define config');
check(config, Object);
if (!config.listConfig) {
throw new Meteor.Error('Error', 'Impossible de définir les configurations');
}
console.log(config);
check(config, Object);
if (!config.userId) {
console.log('add userId');
config.userId = Meteor.userId();
}
if (!config.listConfig) {
config.listConfig = [];
}
_.forEach(config.listConfig, (item) => {
if (item.forbiddenWordString) {
item.forbiddenWord = item.forbiddenWordString.split(',');
} else {
item.forbiddenWord = [];
}
});
configCollection.simpleSchema().validate(config);
configCollection.remove({ userId: Meteor.userId() });
configCollection.insert(config);
configCollection.upsert({ userId: config.userId.toLowerCase() }, {
$set: {
userId: config.userId,
listConfig: config.listConfig,
},
});
}
/**
......@@ -31,7 +40,7 @@ export function defineConfig(config) {
* @returns {any}
*/
export function getConfig() {
return configCollection.findOne({ userId: Meteor.userId() });
return configCollection.findOne({ userId: Meteor.userId().toLowerCase() });
}
Meteor.methods({
......
......@@ -2,4 +2,4 @@
import { Meteor } from 'meteor/meteor';
import configCollection from '/imports/api/config/configCollection';
Meteor.publish('config', () => configCollection.find({ userId: Meteor.userId() }));
Meteor.publish('config', () => Meteor.userId() && configCollection.find({ userId: Meteor.userId().toLowerCase() }));
......@@ -4,6 +4,9 @@ import SimpleSchema from 'simpl-schema';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
name: {
type: String,
},
accessToken: {
type: String,
required: false,
......@@ -41,4 +44,11 @@ export default new SimpleSchema({
label: false,
},
},
typeCrawl: {
type: String,
defaultValue: 'api',
autoform: {
type: 'hidden',
},
},
}, { tracker: Tracker });
......@@ -70,8 +70,8 @@ export default class CrawlFacebook {
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: this.config.userId,
_index: this.config.userId,
_type: 'api',
_id: dataForIndex.url,
},
});
......
......@@ -65,8 +65,8 @@ export default class CrawlTwitter {
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: this.config.userId,
_index: this.config.userId,
_type: 'api',
_id: dataForIndex.url,
},
});
......
......@@ -9,6 +9,7 @@ const hooksFormNetworkCrawl = {
onSubmit(insertDoc, updateDoc, currentDoc) {
const templateInstance = this.template.parent();
const listAlertNetwork = [];
const listFilename = [];
if (insertDoc) {
const files = this.event.currentTarget.uploadFile.files;
const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : '';
......@@ -19,6 +20,7 @@ const hooksFormNetworkCrawl = {
if (files.length > index) {
const file = files[index];
console.log(file.name);
listFilename.push(file.name);
// upload only if file has exist but with modif or not exist (DONT WORK NOW BECAUSE updatedAt NOT EXIST IN COLLECTION FILES)
if (new RegExp(`.(${extensions})`, 'i').test(file.name) || Files.findOne({ name: file.name })) {
file.updatedAt = file.lastModifiedDate;
......@@ -59,9 +61,12 @@ const hooksFormNetworkCrawl = {
iterate();
}
} else {
console.log('indexation');
Meteor.callPromise('indexNetwork')
.then((result) => {
Meteor.callPromise('addJob', {
typeCrawl: insertDoc.typeCrawl,
name: insertDoc.name,
scheduledDate: insertDoc.scheduledDate,
listFilename,
}).then((result) => {
this.done(null, result);
}).catch((error) => {
this.done(error);
......@@ -77,7 +82,7 @@ const hooksFormNetworkCrawl = {
displayNotif({
type: 'success',
title: 'uploadNetwork',
message: result.message,
message: 'Job ajouté avec succès!',
save: true,
});
},
......
import SimpleSchema from 'simpl-schema';
import moment from 'moment';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
name: {
type: String,
label: 'Nom de l\'indexation réseau',
},
pathDirectory: {
type: String,
required: false,
......@@ -11,7 +16,6 @@ export default new SimpleSchema({
},
fileType: {
type: Array,
required: false,
label: 'Type de fichier :',
autoform: {
type: 'select-checkbox-inline',
......@@ -45,5 +49,37 @@ export default new SimpleSchema({
blackbox: true,
required: false,
},
typeCrawl: {
type: String,
defaultValue: 'network',
autoform: {
type: 'hidden',
},
},
scheduledDate: {
type: Date,
label: 'Date d\'éxécution :',
autoform: {
afFieldInput: {
type: 'bootstrap-datetimepicker',
dateTimePickerOptions: {
minDate: moment().add(5, 'm'),
format: 'LLL',
locale: 'fr',
icons: {
time: 'fa fa-clock-o',
date: 'fa fa-calendar',
up: 'fa fa-chevron-up',
down: 'fa fa-chevron-down',
previous: 'fa fa-chevron-left',
next: 'fa fa-chevron-right',
today: 'fa fa-dot-circle-o',
clear: 'fa fa-trash',
close: 'fa fa-times',
},
},
},
},
},
}, { tracker: Tracker });
......@@ -7,8 +7,12 @@ import Files from '/imports/api/crawl/network/networkCollection';
export default class CrawlNetwork extends CrawlGeneric {
constructor() {
constructor(data) {
super();
this.fileToIndex = [];
if (data.listFilename) {
this.fileToIndex = data.listFilename;
}
this.files = Files.find({ userId: Meteor.userId() }).fetch();
this.listDataForIndex = [];
console.log('init crawl network!');
......@@ -22,10 +26,14 @@ export default class CrawlNetwork extends CrawlGeneric {
iterateFile() {
return new Promise((resolve) => {
_.forEach(this.files, (file, index) => {
/**
* indexe seulement les fichiers que l'on veut indexer
*/
if (_.includes(this.fileToIndex, file.name)) {
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: this.config.userId,
_index: this.config.userId,
_type: 'document',
_id: file.name,
},
});
......@@ -35,7 +43,7 @@ export default class CrawlNetwork extends CrawlGeneric {
data: this.base64Encode(file.path),
createdAt: new Date(),
});
}
if (this.files.length === index + 1) {
resolve(this.listDataForIndex);
}
......
......@@ -7,7 +7,7 @@ const hooksFormWebsiteCrawl = {
displayNotif({
type: 'success',
title: 'Indexation : ',
message: result.message,
message: 'Ajouté à la liste de job',
save: true,
});
},
......
import SimpleSchema from 'simpl-schema';
import moment from 'moment';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
name: {
type: String,
},
urlWebsite: {
type: String,
label: 'Url du site ou du sitemap à indexer :',
autoValue() {
return this.value.replace('https', 'http');
},
},
nameConfig: {
type: String,
label: 'Configuration à appliquer :',
},
typeCrawl: {
type: String,
defaultValue: 'website',
autoform: {
type: 'hidden',
},
},
scheduledDate: {
type: Date,
label: 'Date d\'éxécution :',
autoform: {
afFieldInput: {
type: 'bootstrap-datetimepicker',
dateTimePickerOptions: {
minDate: moment().add(5, 'm'),
format: 'LLL',
locale: 'fr',
icons: {
time: 'fa fa-clock-o',
date: 'fa fa-calendar',
up: 'fa fa-chevron-up',
down: 'fa fa-chevron-down',
previous: 'fa fa-chevron-left',
next: 'fa fa-chevron-right',
today: 'fa fa-dot-circle-o',
clear: 'fa fa-trash',
close: 'fa fa-times',
},
},
},
},
},
}, { tracker: Tracker });
......@@ -6,6 +6,7 @@ import { Meteor } from 'meteor/meteor';
import Crawler from 'crawler';
import Sitemapper from 'sitemapper';
import checkData from '/imports/utils/checkData';
import jobCollection from '/imports/api/job/jobCollection';
export default class crawlWebsite extends CrawlGeneric {
......@@ -17,6 +18,7 @@ export default class crawlWebsite extends CrawlGeneric {
*/
constructor(data, contextIndex) {
super();
this.name = data.name;
this.contextIndex = contextIndex;
this.urlWebsite = data.urlWebsite;
this.config.crawl = _.find(this.config.listConfig, n => n.domain === data.nameConfig);
......@@ -68,7 +70,7 @@ export default class crawlWebsite extends CrawlGeneric {
// check if already visited
if (!this.listUrlAlreadyVisited.includes(toQueueUrl)) {
// check if url has forbidden word
if (!new RegExp(this.config.crawl.forbiddenWord.join('|')).test(toQueueUrl)) {
if (this.config.crawl.forbiddenWord.length === 0 || !new RegExp(this.config.crawl.forbiddenWord.join('|')).test(toQueueUrl)) {
// check if url is good for crawl
if (checkData.checkCrawlUrl(toQueueUrl)) {
// add url in already visited urls
......@@ -77,7 +79,6 @@ export default class crawlWebsite extends CrawlGeneric {
if (toQueueUrl.match(/(pdf)/)) {
listPdf.push(toQueueUrl);
} else {
// console.log(toQueueUrl);
crawl.queue(toQueueUrl);
}
}
......@@ -88,7 +89,16 @@ export default class crawlWebsite extends CrawlGeneric {
});
// if url has not parameter => parse data
if (!/(#.*|\?.*)/g.test(res.options.uri)) {
this.parseData(res.$, res.options.uri, listPdf);
this.parseData(res.$, res.options.uri, listPdf)
.then(() => {
jobCollection.update({ name: this.name },
{
$inc: {
numberIndexed: 1,
},
},
);
});
}
}
......@@ -115,7 +125,7 @@ export default class crawlWebsite extends CrawlGeneric {
body.html(checkData.cleanHtml(body.html()));
const title = checkData.cleanText($('title').text());
const dataForIndex = {
tag: 'site',
tag: 'website',
domain: this.config.crawl.domain,
title,
title_suggest: {
......@@ -145,8 +155,8 @@ export default class crawlWebsite extends CrawlGeneric {
dataForIndex.listPdf = listPdf.join(' ');
}
const _index = Meteor.settings.private.elasticsearch.esIndex;
const _type = this.config.userId;
const _index = this.config.userId;
const _type = 'website';
const _id = currentUrl;
this.listDataForIndex.push(dataForIndex);
......
......@@ -15,10 +15,11 @@ export function initIndexElastic() {
return index.initElastic();
}
export function reIndexElastic() {
export function reIndexElastic(esType) {
check(esType, String);
this.unblock();
const index = new IndexGeneric();
return index.reIndex(Meteor.userId());
return index.reIndex(esType);
}
export function indexWebsite(data) {
......@@ -35,9 +36,10 @@ export function indexApi(data) {
return index.start(data);
}
export function indexNetwork() {
export function indexNetwork(data) {
check(data, Object);
const index = new IndexNetwork();
return index.start();
return index.start(data);
}
export function removeFileNetwork(id) {
......@@ -48,7 +50,7 @@ export function removeFileNetwork(id) {
Meteor.methods({
initIndexElastic,
reIndexElastic,
indexWebsite,
// indexWebsite,
indexApi,
indexNetwork,
removeFileNetwork,
......
import CrawlApi from '/imports/api/crawl/api/server/crawlApi';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import jobCollection from '/imports/api/job/jobCollection';
export default class IndexApi extends IndexGeneric {
......@@ -13,6 +14,7 @@ export default class IndexApi extends IndexGeneric {
try {
const dataToIndex = await new CrawlApi(data);
const resultIndex = await this.indexByBulk(dataToIndex);
jobCollection.update({ name: data.name }, { $set: { numberIndexed: resultIndex.items.length } }, () => {});
return {
message: `Indexation ${data.type} de ${resultIndex.items.length} liens pour ${data.idPage} fini avec succès!`,
};
......
import { Meteor } from 'meteor/meteor';
import elastic from '/imports/libs/elasticsearch/elasticsearch';
import configElastic from '/imports/libs/elasticsearch/elasticSearchConfig';
import { getConfig } from '/imports/api/config/methods';
const esIndex = Meteor.settings.private.elasticsearch.esIndex;
export default class IndexGeneric {
constructor() {
this.config = getConfig();
console.log(this.config);
}
indexOne(esIndex, esType, id, obj) {
......@@ -21,18 +17,29 @@ export default class IndexGeneric {
return elastic.bulk(data, hasFile);
}
reIndex(esType) {
const body = {
async reIndex(esType) {
const indexTmp = {
source: {
index: this.config.userId,
type: esType,
},
dest: {
index: `${this.config.userId}1`,
type: esType,
},
};
return elastic.reIndex(indexTmp);
/* const reindexBody = {
source: {
index: 'idsearch',
index: `${this.config.userId}1`,
type: esType,
},
dest: {
index: 'idsearch',
index: this.config.userId,
type: esType,
},
};
return elastic.reIndex(body);
return elastic.reIndex(reindexBody); */
}
/**
......@@ -47,8 +54,10 @@ export default class IndexGeneric {
* (chaque user à son propre type sur elastic)
* initialiser l'analyzer supprimerai egalement le mapping des autres
*/
// await elastic.initAnalyzer(esIndex, analyser);
await elastic.initMapping(esIndex, this.config.userId, mapping);
await elastic.initAnalyzer(this.config.userId, analyser);
await elastic.initMapping(this.config.userId, 'website', mapping);
await elastic.initMapping(this.config.userId, 'api', mapping);
await elastic.initMapping(this.config.userId, 'document', mapping);
return elastic.createPipeline();
}
}
......@@ -4,9 +4,9 @@ import CrawlNetwork from '/imports/api/crawl/network/server/crawlNetwork';
export default class IndexNetwork extends IndexGeneric {
async start() {
async start(data) {
try {
const dataToIndex = await new CrawlNetwork();
const dataToIndex = await new CrawlNetwork(data);
console.log('start indexation network');
const resultIndex = await this.indexByBulk(dataToIndex, true);
return {
......
......@@ -11,7 +11,8 @@ export default class IndexWebsite extends IndexGeneric {
*/
async start(data) {
try {
const dataToIndex = await new CrawlWebsite(data, this);
const context = this;
const dataToIndex = await new CrawlWebsite(data, context);
/**
* dataToIndex index website page
* false no file
......
......@@ -11,11 +11,8 @@ const jobsCollection = new Mongo.Collection('jobs');
SimpleSchema.jobsCollection = new SimpleSchema({
status: {
type: String,
allowedValues: ['running', 'stop'],
/* autoValue() {
return 'stop';
},*/
defaultValue: 'stop',
allowedValues: ['running', 'stopped'],
defaultValue: 'stopped',
},
type: {
type: String,
......@@ -29,6 +26,10 @@ SimpleSchema.jobsCollection = new SimpleSchema({
blackbox: true,
defaultValue: {},
},
numberIndexed: {
type: Number,
defaultValue: 0,
},
scheduledDate: {
type: Date,
required: false,
......@@ -40,12 +41,6 @@ SimpleSchema.jobsCollection = new SimpleSchema({
userId: {
type: String,
},
createdAt: {