Commit 22f1647e authored by Nacim Goura's avatar Nacim Goura

add upload and index file

parent 8a4d0043
......@@ -35,6 +35,7 @@
"no-underscore-dangle": "off",
"class-methods-use-this": "off",
"no-param-reassign": "off",
"meteor/no-session": "off"
"meteor/no-session": "off",
"no-console": "off"
}
}
......@@ -7,7 +7,7 @@
# base package
meteor-base@1.1.0 # Packages every Meteor app needs to have
mobile-experience@1.0.4 # Packages for a great mobile UX
mongo@1.1.19-rc.1 # The database Meteor supports right now
mongo@1.1.19-rc.3 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library
......@@ -15,7 +15,7 @@ dynamic-import@0.1.1
# package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode
standard-minifier-js@2.1.1-rc.1 # JS minifier run for production mode
standard-minifier-js@2.1.1-rc.3 # JS minifier run for production mode
es5-shim@4.6.15 # ECMAScript 5 compatibility for older browsers.
ecmascript@0.8.1 # Enable ECMAScript2015+ syntax in app code
fourseven:scss@4.5.0 # Compile scss to css
......@@ -40,14 +40,13 @@ aldeed:autoform
matb33:collection-hooks
# package for user
accounts-base@1.3.1-rc.1
accounts-password@1.4.0-rc.1
accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.3
alanning:roles
# other packages
deanius:promise
aldeed:tabular
percolate:synced-cron
qnipp:treeview
ostrio:files
aldeed:template-extension
METEOR@1.5.1-rc.1
METEOR@1.5.1-rc.3
accounts-base@1.3.1-rc.1
accounts-password@1.4.0-rc.1
accounts-base@1.3.1-rc.3
accounts-password@1.4.0-rc.3
ajduke:bootstrap-tagsinput@0.7.1
alanning:roles@1.2.16
aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1
aldeed:template-extension@4.1.0
allow-deny@1.0.6-rc.1
allow-deny@1.0.6-rc.3
autoupdate@1.3.12
babel-compiler@6.19.3
babel-runtime@1.0.1
......@@ -15,7 +15,7 @@ binary-heap@1.0.10
blaze@2.3.2
blaze-html-templates@1.1.2
blaze-tools@1.0.10
boilerplate-generator@1.1.0
boilerplate-generator@1.1.1
caching-compiler@1.1.9
caching-html-compiler@1.1.2
callback-hook@1.0.10
......@@ -23,11 +23,11 @@ check@1.2.5
coffeescript@1.12.6_1
cosmos:browserify@0.10.0
dburles:collection-helpers@1.1.0
ddp@1.3.0-rc.1
ddp-client@2.0.0-rc.1
ddp-common@1.2.9-rc.1
ddp@1.3.0-rc.3
ddp-client@2.0.0-rc.3
ddp-common@1.2.9-rc.3
ddp-rate-limiter@1.0.7
ddp-server@2.0.0-rc.1
ddp-server@2.0.0-rc.3
deanius:promise@3.1.3
deps@1.0.12
diff-sequence@1.0.7
......@@ -57,17 +57,17 @@ livedata@1.0.18
localstorage@1.1.0
logging@1.1.17
matb33:collection-hooks@0.8.4
meteor@1.7.0-rc.1
meteor@1.7.0-rc.3
meteor-base@1.1.0
minifier-css@1.2.16
minifier-js@2.1.1-rc.1
minifier-js@2.1.1-rc.3
minimongo@1.2.1
mobile-experience@1.0.4
mobile-status-bar@1.0.14
modules@0.9.2
modules-runtime@0.8.0
momentjs:moment@2.18.1
mongo@1.1.19-rc.1
mongo@1.1.19-rc.3
mongo-id@1.0.6
npm-bcrypt@0.9.3
npm-mongo@2.2.24
......@@ -82,8 +82,6 @@ practicalmeteor:mocha@2.4.5_6
practicalmeteor:mocha-core@1.0.1
practicalmeteor:sinon@1.14.1_2
promise@0.8.9
qnipp:jstree@3.3.0
qnipp:treeview@1.0.0
raix:eventemitter@0.1.3
random@1.0.10
rate-limit@1.0.8
......@@ -99,7 +97,7 @@ spacebars@1.0.15
spacebars-compiler@1.1.2
srp@1.0.10
standard-minifier-css@1.3.4
standard-minifier-js@2.1.1-rc.1
standard-minifier-js@2.1.1-rc.3
templating@1.3.2
templating-compiler@1.3.2
templating-runtime@1.3.2
......@@ -111,5 +109,5 @@ twbs:bootstrap@3.3.6
ui@1.0.13
underscore@1.0.10
url@1.1.0
webapp@1.3.17-rc.1
webapp@1.3.17-rc.3
webapp-hashing@1.0.9
import SimpleSchema from 'simpl-schema';
import { Tracker } from 'meteor/tracker';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
email: {
type: String,
regEx: SimpleSchema.RegEx.Email,
},
username: {
type: String,
},
password: {
type: String,
},
}, { tracker: Tracker });
import SimpleSchema from 'simpl-schema';
import formAccountSchema from '/imports/api/account/formAccountSchema';
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import { Accounts } from 'meteor/accounts-base';
/**
* add user account
* @param user
*/
export function addAccount(user) {
check(user, Object);
formAccountSchema.validate(user);
const id = Accounts.createUser(user);
if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter!');
}
}
/**
* delete user account
* @param id
*/
export function deleteAccount(id) {
check(id, String);
Meteor.users.remove({ _id: id });
}
Meteor.methods({
// add account
addAccount(user) {
// validate user data
check(user, Object);
new SimpleSchema({
email: {
type: String,
regEx: SimpleSchema.RegEx.Email,
},
username: String,
password: String,
}).validate(user);
const id = Accounts.createUser(user);
if (!id) {
throw new Meteor.Error('Error', 'Impossible de se connecter');
}
},
// delete account
deleteAccount(id) {
check(id, String);
Meteor.users.remove({ _id: id });
},
addAccount,
deleteAccount,
});
import SimpleSchema from 'simpl-schema';
import { Mongo } from 'meteor/mongo';
import { Tracker } from 'meteor/tracker';
SimpleSchema.extendOptions(['autoform']);
......@@ -17,17 +18,19 @@ SimpleSchema.configCollection = new SimpleSchema({
domain: {
type: String,
regEx: SimpleSchema.RegEx.Url,
required: false,
},
breadcrumb: {
type: String,
},
forbiddenWord: {
type: Array,
required: false,
},
'forbiddenWord.$': {
type: String,
},
});
}, { tracker: Tracker });
configCollection.attachSchema(SimpleSchema.configCollection);
......
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import configCollection from '/imports/api/config/configCollection';
/**
* define config for user
* @param config
*/
export function defineConfig(config) {
check(config, Object);
configCollection.validate(config);
const oldConfig = configCollection.find({ userId: Meteor.userId() }).fetch();
if (config.forbiddenWord) {
config.forbiddenWord = config.forbiddenWord.split(',');
......@@ -28,6 +33,10 @@ export function defineConfig(config) {
}
}
/**
* get config for user
* @returns {any}
*/
export function getConfig() {
return configCollection.findOne({ userId: Meteor.userId() });
}
......
import { AutoForm } from 'meteor/aldeed:autoform';
import displayNotif from '/imports/api/notif/notifClient';
const hooksFormApiCrawl = {
onSuccess(formType, result) {},
onError(formType, error) {},
before: {
method: (doc) => {
displayNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexApi',
message: `Début de l'indexation ${doc.type} pour ${doc.idPage}!`,
save: true,
});
return doc;
},
},
onSuccess(formType, result) {
displayNotif({
userId: Meteor.userId(),
type: 'success',
title: 'indexApi',
message: result.message,
save: true,
});
},
onError(formType, error) {
displayNotif({
userId: Meteor.userId(),
type: 'error',
title: 'indexApi',
message: error ? JSON.stringify(error.message) : 'Errror Index API',
save: true,
});
},
};
AutoForm.hooks({
......
......@@ -4,11 +4,7 @@ import CrawlTwitter from './crawlTwitter';
export default class crawlApi {
constructor() {
console.log('init crawl api!');
}
init(config) {
constructor(config) {
switch (config.type) {
case 'facebook':
return new CrawlFacebook(config);
......
import { Meteor } from 'meteor/meteor';
import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
import Files from '/imports/api/crawl/network/networkCollection';
const hooksFormNetworkCrawl = {
onSubmit: function (insertDoc, updateDoc, currentDoc) {
onSubmit(insertDoc, updateDoc, currentDoc) {
const templateInstance = this.template.parent();
const listAlertNetwork = [];
if (insertDoc) {
......@@ -13,48 +13,81 @@ const hooksFormNetworkCrawl = {
const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : '';
if (files && extensions) {
console.log(extensions);
_.each(files, function (file) {
file.allowedExtensions = extensions;
const upload = Files.insert({
file,
streams: 'dynamic',
chunkSize: 'dynamic',
}, false);
let index = 0;
const iterate = () => {
if (files.length > index) {
const file = files[index];
console.log(file.name);
// upload only if file has exist but with modif or not exist (DONT WORK NOW BECAUSE updatedAt NOT EXIST IN COLLECTION FILES)
if (new RegExp(`.(${extensions})`, 'i').test(file.name) || Files.findOne({ name: file.name })) {
file.updatedAt = file.lastModifiedDate;
const upload = Files.insert({
file,
streams: 'dynamic',
chunkSize: 'dynamic',
}, false);
upload.on('start', function () {
console.log('start upload!');
templateInstance.currentUpload.set(this);
});
upload.on('start', function () {
console.log('start upload!');
templateInstance.currentUpload.set(this);
});
upload.on('end', (error, fileObj) => {
console.log(fileObj.name);
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
});
} else {
listAlertNetwork.push({
type: 'alert-success',
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
}
templateInstance.numberFileUploaded.set(templateInstance.numberFileUploaded.get() + 1);
templateInstance.listAlertNetwork.set(listAlertNetwork);
templateInstance.currentUpload.set(false);
upload.on('end', (error, fileObj) => {
console.log('end upload');
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
index++;
iterate();
});
upload.start();
} else {
listAlertNetwork.push({
type: 'alert-success',
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
index++;
iterate();
}
templateInstance.listAlertNetwork.set(listAlertNetwork);
templateInstance.currentUpload.set(false);
});
upload.start();
});
} else {
console.log('indexation');
Meteor.callPromise('indexNetwork')
.then((result) => {
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'success',
title: 'uploadNetwork',
message: result.message,
save: true,
});
this.done();
}).catch((error) => {
console.log(error);
Meteor.call('addNotif', {
userId: Meteor.userId(),
type: 'error',
title: 'uploadNetwork',
message: JSON.stringify(error),
save: true,
});
this.done();
});
}
};
iterate();
}
}
return false;
},
onSuccess(formType, result) {},
onError(formType, error) {},
};
AutoForm.hooks({
......
import { FilesCollection } from 'meteor/ostrio:files';
import { Mongo } from 'meteor/mongo';
/**
* @type {Mongo.Collection}
*/
const TreeData = new Mongo.Collection('TreeData');
const Files = new FilesCollection({
collectionName: 'Files',
allowClientCode: false, // Disallow remove files from Client
onBeforeUpload(file) {
console.log(file.allowedExtensions, file.extension, new RegExp(file.allowedExtensions, 'i').test(file.extension));
if (file.size > 10485760) {
return `(${file.name}) Please upload file with size equal or less than 10MB`;
}
return new RegExp(file.allowedExtensions, 'i').test(file.extension);
return true;
},
});
export { TreeData, Files };
export default Files;
import { Meteor } from 'meteor/meteor';
import fs from 'fs';
import _ from 'lodash';
import Files from '/imports/api/crawl/network/networkCollection';
export default class CrawlNetwork {
constructor() {
this.files = Files.find({ userId: Meteor.userId() }).fetch();
this.listDataForIndex = [];
console.log('init crawl network!');
return this.start();
}
start() {
return this.iterateFile();
}
iterateFile() {
return new Promise((resolve) => {
_.forEach(this.files, (file, index) => {
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType,
_id: file.name,
},
});
this.listDataForIndex.push({
tag: 'network',
data: this.base64Encode(file.path),
createdAt: new Date(),
});
if (this.files.length === index + 1) {
resolve(this.listDataForIndex);
}
});
});
}
base64Encode(file) {
// read binary data
const bitmap = fs.readFileSync(file);
// convert binary data to base64 encoded string
return new Buffer(bitmap).toString('base64');
}
}
import { AutoForm } from 'meteor/aldeed:autoform';
import { displayNotif } from '/imports/api/notif/methods';
const hooksFormWebsiteCrawl = {
onSuccess(formType, result) {
......
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import Files from '/imports/api/crawl/network/networkCollection';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import { addNotif } from '/imports/api/notif/methods';
Meteor.methods({
initIndexElastic() {
this.unblock();
try {
const index = new IndexGeneric();
index.initElastic();
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'initIndexElastic',
message: 'ElasticSearch à été initialisé correctement!',
save: true,
});
} catch (error) {
addNotif({
userId: Meteor.userId(),
type: 'error',
title: 'initIndexElastic',
message: error,
save: true,
});
}
const index = new IndexGeneric();
return index.initElastic();
},
reIndexElastic() {
this.unblock();
try {
const index = new IndexGeneric();
index.reIndex();
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'reIndexElastic',
message: 'ElasticSearch à été réindexé correctement!',
save: true,
});
} catch (error) {
addNotif({
userId: Meteor.userId(),
type: 'error',
title: 'reIndexElastic',
message: error,
save: true,
});
}
const index = new IndexGeneric();
return index.reIndex();
},
indexWebsite(url) {
check(url, String);
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexWebsite',
message: `Début de l'indexation pour ${url}!`,
save: true,
});
try {
const index = new IndexWebsite();
index.indexationWebsite(url)
.then((result) => {
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexWebsite',
message: result.message,
save: true,
});
});
} catch (error) {
addNotif({
userId: Meteor.userId(),
type: 'error',
title: 'indexWebsite',
message: error,
save: true,
});
}
const index = new IndexWebsite();
return index.start(url);
},
indexApi(data) {
check(data, Object);
formApiCrawlSchema.validate(data);
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexApi',
message: `Début de l'indexation ${data.type} pour ${data.idPage}!`,
save: true,
});
try {
const index = new IndexApi();
index.indexationApi(data)
.then((result) => {
addNotif({
userId: Meteor.userId(),
type: 'info',
title: 'indexApi',
message: result.message,
save: true,
});
});
} catch (error) {
addNotif({
userId: Meteor.userId(),
type: 'error',
title: 'indexApi',
message: error,
save: true,
});
}
const index = new IndexApi();
return index.start(data);
},
// display all file in network
crawlNetwork(data) {
check(data, Object);
// formNetworkCrawlSchema.validate(data);
this.unblock();
indexNetwork() {
const indexNetwork = new IndexNetwork();
indexNetwork.crawlNetwork(data);
return indexNetwork.start();
},
indexNetwork(files) {
check(files, Array);
const indexNetwork = new IndexNetwork();
indexNetwork.initParseFiles(files);
removeFileNetwork(id) {
check(id, String);
Files.remove({ _id: id });
},
});
import { Meteor } from 'meteor/meteor';
import { Files, TreeData } from '/imports/api/crawl/network/networkCollection';
import Files from '/imports/api/crawl/network/networkCollection';
Meteor.publish('TreeData', () => TreeData.find({ userId: Meteor.userId() }));
Meteor.publish('files.images.all', () => Files.find().cursor);
Meteor.publish('filesNetwork', () => Files.find({ userId: Meteor.userId() }).cursor);
import CrawlApi from '/imports/api/crawl/api/server/crawlApi';
import IndexGeneric from './indexGeneric';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
export default class IndexApi extends IndexGeneric {
......@@ -9,16 +9,15 @@ export default class IndexApi extends IndexGeneric {
* @param data
* @returns Promise
*/
async indexationApi(data) {