Commit 22f1647e authored by Nacim Goura's avatar Nacim Goura

add upload and index file

parent 8a4d0043
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
"no-underscore-dangle": "off", "no-underscore-dangle": "off",
"class-methods-use-this": "off", "class-methods-use-this": "off",
"no-param-reassign": "off", "no-param-reassign": "off",
"meteor/no-session": "off" "meteor/no-session": "off",
"no-console": "off"
} }
} }
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
# base package # base package
meteor-base@1.1.0 # Packages every Meteor app needs to have meteor-base@1.1.0 # Packages every Meteor app needs to have
mobile-experience@1.0.4 # Packages for a great mobile UX mobile-experience@1.0.4 # Packages for a great mobile UX
mongo@1.1.19-rc.1 # The database Meteor supports right now mongo@1.1.19-rc.3 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library tracker@1.1.3 # Meteor's client-side reactive programming library
...@@ -15,7 +15,7 @@ dynamic-import@0.1.1 ...@@ -15,7 +15,7 @@ dynamic-import@0.1.1
# package for compile and minify # package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode standard-minifier-css@1.3.4 # CSS minifier run for production mode
standard-minifier-js@2.1.1-rc.1 # JS minifier run for production mode standard-minifier-js@2.1.1-rc.3 # JS minifier run for production mode
es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers. es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers.
ecmascript@0.8.1 # Enable ECMAScript2015+ syntax in app code ecmascript@0.8.1 # Enable ECMAScript2015+ syntax in app code
fourseven:scss@4.5.0 # Compile scss to css fourseven:scss@4.5.0 # Compile scss to css
...@@ -40,14 +40,13 @@ aldeed:autoform ...@@ -40,14 +40,13 @@ aldeed:autoform
matb33:collection-hooks matb33:collection-hooks
# package for user # package for user
accounts-base@1.3.1-rc.1 accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.1 accounts-password@1.4.0-rc.3
alanning:roles alanning:roles
# other packages # other packages
deanius:promise deanius:promise
aldeed:tabular aldeed:tabular
percolate:synced-cron percolate:synced-cron
qnipp:treeview
ostrio:files ostrio:files
aldeed:template-extension aldeed:template-extension
METEOR@1.5.1-rc.1 METEOR@1.5.1-rc.3
accounts-base@1.3.1-rc.1 accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.1 accounts-password@1.4.0-rc.3
ajduke:bootstrap-tagsinput@0.7.1 ajduke:bootstrap-tagsinput@0.7.1
alanning:roles@1.2.16 alanning:roles@1.2.16
aldeed:autoform@6.2.0 aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1 aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1 aldeed:tabular@2.1.1
aldeed:template-extension@4.1.0 aldeed:template-extension@4.1.0
allow-deny@1.0.6-rc.1 allow-deny@1.0.6-rc.3
autoupdate@1.3.12 autoupdate@1.3.12
babel-compiler@6.19.3 babel-compiler@6.19.3
babel-runtime@1.0.1 babel-runtime@1.0.1
...@@ -15,7 +15,7 @@ binary-heap@1.0.10 ...@@ -15,7 +15,7 @@ binary-heap@1.0.10
blaze@2.3.2 blaze@2.3.2
blaze-html-templates@1.1.2 blaze-html-templates@1.1.2
blaze-tools@1.0.10 blaze-tools@1.0.10
boilerplate-generator@1.1.0 boilerplate-generator@1.1.1
caching-compiler@1.1.9 caching-compiler@1.1.9
caching-html-compiler@1.1.2 caching-html-compiler@1.1.2
callback-hook@1.0.10 callback-hook@1.0.10
...@@ -23,11 +23,11 @@ check@1.2.5 ...@@ -23,11 +23,11 @@ check@1.2.5
coffeescript@1.12.6_1 coffeescript@1.12.6_1
cosmos:browserify@0.10.0 cosmos:browserify@0.10.0
dburles:collection-helpers@1.1.0 dburles:collection-helpers@1.1.0
ddp@1.3.0-rc.1 ddp@1.3.0-rc.3
ddp-client@2.0.0-rc.1 ddp-client@2.0.0-rc.3
ddp-common@1.2.9-rc.1 ddp-common@1.2.9-rc.3
ddp-rate-limiter@1.0.7 ddp-rate-limiter@1.0.7
ddp-server@2.0.0-rc.1 ddp-server@2.0.0-rc.3
deanius:promise@3.1.3 deanius:promise@3.1.3
deps@1.0.12 deps@1.0.12
diff-sequence@1.0.7 diff-sequence@1.0.7
...@@ -57,17 +57,17 @@ livedata@1.0.18 ...@@ -57,17 +57,17 @@ livedata@1.0.18
localstorage@1.1.0 localstorage@1.1.0
logging@1.1.17 logging@1.1.17
matb33:collection-hooks@0.8.4 matb33:collection-hooks@0.8.4
meteor@1.7.0-rc.1 meteor@1.7.0-rc.3
meteor-base@1.1.0 meteor-base@1.1.0
minifier-css@1.2.16 minifier-css@1.2.16
minifier-js@2.1.1-rc.1 minifier-js@2.1.1-rc.3
minimongo@1.2.1 minimongo@1.2.1
mobile-experience@1.0.4 mobile-experience@1.0.4
mobile-status-bar@1.0.14 mobile-status-bar@1.0.14
modules@0.9.2 modules@0.9.2
modules-runtime@0.8.0 modules-runtime@0.8.0
momentjs:moment@2.18.1 momentjs:moment@2.18.1
mongo@1.1.19-rc.1 mongo@1.1.19-rc.3
mongo-id@1.0.6 mongo-id@1.0.6
npm-bcrypt@0.9.3 npm-bcrypt@0.9.3
npm-mongo@2.2.24 npm-mongo@2.2.24
...@@ -82,8 +82,6 @@ practicalmeteor:mocha@2.4.5_6 ...@@ -82,8 +82,6 @@ practicalmeteor:mocha@2.4.5_6
practicalmeteor:mocha-core@1.0.1 practicalmeteor:mocha-core@1.0.1
practicalmeteor:sinon@1.14.1_2 practicalmeteor:sinon@1.14.1_2
promise@0.8.9 promise@0.8.9
qnipp:jstree@3.3.0
qnipp:treeview@1.0.0
raix:eventemitter@0.1.3 raix:eventemitter@0.1.3
random@1.0.10 random@1.0.10
rate-limit@1.0.8 rate-limit@1.0.8
...@@ -99,7 +97,7 @@ spacebars@1.0.15 ...@@ -99,7 +97,7 @@ spacebars@1.0.15
spacebars-compiler@1.1.2 spacebars-compiler@1.1.2
srp@1.0.10 srp@1.0.10
standard-minifier-css@1.3.4 standard-minifier-css@1.3.4
standard-minifier-js@2.1.1-rc.1 standard-minifier-js@2.1.1-rc.3
templating@1.3.2 templating@1.3.2
templating-compiler@1.3.2 templating-compiler@1.3.2
templating-runtime@1.3.2 templating-runtime@1.3.2
...@@ -111,5 +109,5 @@ twbs:bootstrap@3.3.6 ...@@ -111,5 +109,5 @@ twbs:bootstrap@3.3.6
ui@1.0.13 ui@1.0.13
underscore@1.0.10 underscore@1.0.10
url@1.1.0 url@1.1.0
webapp@1.3.17-rc.1 webapp@1.3.17-rc.3
webapp-hashing@1.0.9 webapp-hashing@1.0.9
import SimpleSchema from 'simpl-schema';
import { Tracker } from 'meteor/tracker';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
email: {
type: String,
regEx: SimpleSchema.RegEx.Email,
},
username: {
type: String,
},
password: {
type: String,
},
}, { tracker: Tracker });
import SimpleSchema from 'simpl-schema'; import formAccountSchema from '/imports/api/account/formAccountSchema';
import { check } from 'meteor/check'; import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { Accounts } from 'meteor/accounts-base'; import { Accounts } from 'meteor/accounts-base';
/**
* add user account
* @param user
*/
export function addAccount(user) {
check(user, Object);
formAccountSchema.validate(user);
const id = Accounts.createUser(user);
if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter!');
}
}
/**
* delete user account
* @param id
*/
export function deleteAccount(id) {
check(id, String);
Meteor.users.remove({ _id: id });
}
Meteor.methods({ Meteor.methods({
// add account addAccount,
addAccount(user) { deleteAccount,
// validate user data
check(user, Object);
new SimpleSchema({
email: {
type: String,
regEx: SimpleSchema.RegEx.Email,
},
username: String,
password: String,
}).validate(user);
const id = Accounts.createUser(user);
if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter');
}
},
// delete account
deleteAccount(id) {
check(id, String);
Meteor.users.remove({ _id: id });
},
}); });
import SimpleSchema from 'simpl-schema'; import SimpleSchema from 'simpl-schema';
import { Mongo } from 'meteor/mongo'; import { Mongo } from 'meteor/mongo';
import { Tracker } from 'meteor/tracker';
SimpleSchema.extendOptions(['autoform']); SimpleSchema.extendOptions(['autoform']);
...@@ -17,17 +18,19 @@ SimpleSchema.configCollection = new SimpleSchema({ ...@@ -17,17 +18,19 @@ SimpleSchema.configCollection = new SimpleSchema({
domain: { domain: {
type: String, type: String,
regEx: SimpleSchema.RegEx.Url, regEx: SimpleSchema.RegEx.Url,
required: false,
}, },
breadcrumb: { breadcrumb: {
type: String, type: String,
}, },
forbiddenWord: { forbiddenWord: {
type: Array, type: Array,
required: false,
}, },
'forbiddenWord.$': { 'forbiddenWord.$': {
type: String, type: String,
}, },
}); }, { tracker: Tracker });
configCollection.attachSchema(SimpleSchema.configCollection); configCollection.attachSchema(SimpleSchema.configCollection);
......
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import configCollection from '/imports/api/config/configCollection'; import configCollection from '/imports/api/config/configCollection';
/**
* define config for user
* @param config
*/
export function defineConfig(config) { export function defineConfig(config) {
check(config, Object); check(config, Object);
configCollection.validate(config);
const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch(); const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch();
if (config.forbiddenWord) { if (config.forbiddenWord) {
config.forbiddenWord = config.forbiddenWord.split(','); config.forbiddenWord = config.forbiddenWord.split(',');
...@@ -28,6 +33,10 @@ export function defineConfig(config) { ...@@ -28,6 +33,10 @@ export function defineConfig(config) {
} }
} }
/**
* get config for user
* @returns {any}
*/
export function getConfig() { export function getConfig() {
return configCollection.findOne({ userId: Meteor.userId() }); return configCollection.findOne({ userId: Meteor.userId() });
} }
......
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormApiCrawl = { const hooksFormApiCrawl = {
onSuccess(formType, result) {}, before: {
onError(formType, error) {}, method: (doc) => {
displayNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexApi',
message: `Début de l'indexation ${doc.type} pour ${doc.idPage}!`,
save: true,
});
return doc;
},
},
onSuccess(formType, result) {
displayNotif({
userId: Meteor.userId(),
type: 'success',
title: 'indexApi',
message: result.message,
save: true,
});
},
onError(formType, error) {
displayNotif({
userId: Meteor.userId(),
type: 'error',
title: 'indexApi',
message: error ? JSON.stringify(error.message) : 'Errror Index API',
save: true,
});
},
}; };
AutoForm.hooks({ AutoForm.hooks({
......
...@@ -4,11 +4,7 @@ import CrawlTwitter from './crawlTwitter'; ...@@ -4,11 +4,7 @@ import CrawlTwitter from './crawlTwitter';
export default class crawlApi { export default class crawlApi {
constructor() { constructor(config) {
console.log('init crawl api!');
}
init(config) {
switch (config.type) { switch (config.type) {
case 'facebook': case 'facebook':
return new CrawlFacebook(config); return new CrawlFacebook(config);
......
import { Meteor } from 'meteor/meteor';
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore'; import { _ } from 'meteor/underscore';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection'; import Files from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
const hooksFormNetworkCrawl = { const hooksFormNetworkCrawl = {
onSubmit: function (insertDoc, updateDoc, currentDoc) { onSubmit(insertDoc, updateDoc, currentDoc) {
const templateInstance = this.template.parent(); const templateInstance = this.template.parent();
const listAlertNetwork = []; const listAlertNetwork = [];
if (insertDoc) { if (insertDoc) {
...@@ -13,48 +13,81 @@ const hooksFormNetworkCrawl = { ...@@ -13,48 +13,81 @@ const hooksFormNetworkCrawl = {
const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : ''; const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : '';
if (files && extensions) { if (files && extensions) {
console.log(extensions); let index = 0;
_.each(files, function (file) { const iterate = () => {
file.allowedExtensions = extensions; if (files.length > index) {
const upload = Files.insert({ const file = files[index];
file, console.log(file.name);
streams: 'dynamic', // upload only if file has exist but with modif or not exist (DONT WORK NOW BECAUSE updatedAt NOT EXIST IN COLLECTION FILES)
chunkSize: 'dynamic', if (new RegExp(`.(${extensions})`, 'i').test(file.name) || Files.findOne({ name: file.name })) {
}, false); file.updatedAt = file.lastModifiedDate;
const upload = Files.insert({
file,
streams: 'dynamic',
chunkSize: 'dynamic',
}, false);
upload.on('start', function () {
console.log('start upload!');
templateInstance.currentUpload.set(this);
});
upload.on('start', function () { upload.on('end', (error, fileObj) => {
console.log('start upload!'); console.log(fileObj.name);
templateInstance.currentUpload.set(this); if (error) {
}); listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
});
} else {
listAlertNetwork.push({
type: 'alert-success',
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
}
templateInstance.numberFileUploaded.set(templateInstance.numberFileUploaded.get() + 1);
templateInstance.listAlertNetwork.set(listAlertNetwork);
templateInstance.currentUpload.set(false);
upload.on('end', (error, fileObj) => { index++;
console.log('end upload'); iterate();
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
}); });
upload.start();
} else { } else {
listAlertNetwork.push({ index++;
type: 'alert-success', iterate();
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
} }
templateInstance.listAlertNetwork.set(listAlertNetwork); } else {
templateInstance.currentUpload.set(false); console.log('indexation');
}); Meteor.callPromise('indexNetwork')
.then((result) => {
upload.start(); Meteor.call('addNotif', {
}); userId: Meteor.userId(),
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
this.done();
}).catch((error) => {
console.log(error);
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
this.done();
});
}
};
iterate();
} }
} }
return false; return false;
}, },
onSuccess(formType, result) {},
onError(formType, error) {},
}; };
AutoForm.hooks({ AutoForm.hooks({
......
import { FilesCollection } from 'meteor/ostrio:files'; import { FilesCollection } from 'meteor/ostrio:files';
import { Mongo } from 'meteor/mongo';
/**
* @type {Mongo.Collection}
*/
const TreeData = new Mongo.Collection('TreeData');
const Files = new FilesCollection({ const Files = new FilesCollection({
collectionName: 'Files', collectionName: 'Files',
allowClientCode: false, // Disallow remove files from Client allowClientCode: false, // Disallow remove files from Client
onBeforeUpload(file) { onBeforeUpload(file) {
console.log(file.allowedExtensions, file.extension, new RegExp(file.allowedExtensions, 'i').test(file.extension));
if (file.size > 10485760) { if (file.size > 10485760) {
return `(${file.name}) Please upload file with size equal or less than 10MB`; return `(${file.name}) Please upload file with size equal or less than 10MB`;
} }
return new RegExp(file.allowedExtensions, 'i').test(file.extension); return true;
}, },
}); });
export { TreeData, Files }; export default Files;
import { Meteor } from 'meteor/meteor';
import fs from 'fs';
import _ from 'lodash';
import Files from '/imports/api/crawl/network/networkCollection';
export default class CrawlNetwork {
constructor() {
this.files = Files.find({ userId: Meteor.userId() }).fetch();
this.listDataForIndex = [];
console.log('init crawl network!');
return this.start();
}
start() {
return this.iterateFile();
}
iterateFile() {
return new Promise((resolve) => {
_.forEach(this.files, (file, index) => {
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType,
_id: file.name,
},
});
this.listDataForIndex.push({
tag: 'network',
data: this.base64Encode(file.path),
createdAt: new Date(),
});
if (this.files.length === index + 1) {
resolve(this.listDataForIndex);
}
});
});
}
base64Encode(file) {
// read binary data
const bitmap = fs.readFileSync(file);
// convert binary data to base64 encoded string
return new Buffer(bitmap).toString('base64');
}
}
import { AutoForm } from 'meteor/aldeed:autoform'; import { AutoForm } from 'meteor/aldeed:autoform';
import { displayNotif } from '/imports/api/notif/methods';
const hooksFormWebsiteCrawl = { const hooksFormWebsiteCrawl = {
onSuccess(formType, result) { onSuccess(formType, result) {
......
import { check } from 'meteor/check'; import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import Files from '/imports/api/crawl/network/networkCollection';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema'; import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric'; import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite'; import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi'; import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexNetwork from '/imports/api/indexation/server/indexNetwork'; import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import { addNotif } from '/imports/api/notif/methods';
Meteor.methods({ Meteor.methods({
initIndexElastic() { initIndexElastic() {
this.unblock(); this.unblock();
try { const index = new IndexGeneric();
const index = new IndexGeneric(); return index.initElastic();
index.initElastic();
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'initIndexElastic',
message: 'ElasticSearch à été initialisé correctement!',
save: true,
});
} catch (error) {
addNotif({
userId: Meteor.userId(),
type: 'error',
title: 'initIndexElastic',
message: error,
save: true,
});
}
}, },
reIndexElastic() {