Commit 8a4d0043 authored by Nacim Goura's avatar Nacim Goura

update crawl network

parent e0544ebf
...@@ -11,6 +11,7 @@ mongo@1.1.19-rc.1 # The database Meteor supports right now ...@@ -11,6 +11,7 @@ mongo@1.1.19-rc.1 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library tracker@1.1.3 # Meteor's client-side reactive programming library
dynamic-import@0.1.1
# package for compile and minify # package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode standard-minifier-css@1.3.4 # CSS minifier run for production mode
...@@ -23,29 +24,30 @@ fortawesome:fontawesome # pretty icon ...@@ -23,29 +24,30 @@ fortawesome:fontawesome # pretty icon
# package for view # package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter) kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
ajduke:bootstrap-tagsinput # tag input
# package for test # package for test or validate
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
johanbrook:publication-collector # Test a Meteor publication by collecting its output johanbrook:publication-collector # Test a Meteor publication by collecting its output
practicalmeteor:chai
check@1.2.5
# package for data # package for data
session@1.1.7 session@1.1.7
aldeed:collection2-core aldeed:collection2-core
dburles:collection-helpers dburles:collection-helpers
aldeed:autoform aldeed:autoform
check@1.2.5 matb33:collection-hooks
# package for user # package for user
accounts-base@1.3.1-rc.1 accounts-base@1.3.1-rc.1
accounts-password@1.4.0-rc.1 accounts-password@1.4.0-rc.1
alanning:roles alanning:roles
practicalmeteor:chai
# other packages # other packages
deanius:promise deanius:promise
dynamic-import@0.1.1
aldeed:tabular aldeed:tabular
ajduke:bootstrap-tagsinput
percolate:synced-cron percolate:synced-cron
matb33:collection-hooks
qnipp:treeview qnipp:treeview
ostrio:files
aldeed:template-extension
...@@ -5,6 +5,7 @@ alanning:roles@1.2.16 ...@@ -5,6 +5,7 @@ alanning:roles@1.2.16
aldeed:autoform@6.2.0 aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1 aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1 aldeed:tabular@2.1.1
aldeed:template-extension@4.1.0
allow-deny@1.0.6-rc.1 allow-deny@1.0.6-rc.1
autoupdate@1.3.12 autoupdate@1.3.12
babel-compiler@6.19.3 babel-compiler@6.19.3
...@@ -72,6 +73,8 @@ npm-bcrypt@0.9.3 ...@@ -72,6 +73,8 @@ npm-bcrypt@0.9.3
npm-mongo@2.2.24 npm-mongo@2.2.24
observe-sequence@1.0.16 observe-sequence@1.0.16
ordered-dict@1.0.9 ordered-dict@1.0.9
ostrio:cookies@2.2.1
ostrio:files@1.8.0
percolate:synced-cron@1.3.2 percolate:synced-cron@1.3.2
practicalmeteor:chai@2.1.0_1 practicalmeteor:chai@2.1.0_1
practicalmeteor:loglevel@1.2.0_2 practicalmeteor:loglevel@1.2.0_2
......
import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
const hooksFormNetworkCrawl = {
onSubmit: function (insertDoc, updateDoc, currentDoc) {
const templateInstance = this.template.parent();
const listAlertNetwork = [];
if (insertDoc) {
const files = this.event.currentTarget.uploadFile.files;
const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : '';
if (files && extensions) {
console.log(extensions);
_.each(files, function (file) {
file.allowedExtensions = extensions;
const upload = Files.insert({
file,
streams: 'dynamic',
chunkSize: 'dynamic',
}, false);
upload.on('start', function () {
console.log('start upload!');
templateInstance.currentUpload.set(this);
});
upload.on('end', (error, fileObj) => {
console.log('end upload');
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
});
} else {
listAlertNetwork.push({
type: 'alert-success',
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
}
templateInstance.listAlertNetwork.set(listAlertNetwork);
templateInstance.currentUpload.set(false);
});
upload.start();
});
}
}
return false;
},
onSuccess(formType, result) {},
onError(formType, error) {},
};
AutoForm.hooks({
formNetworkCrawl: hooksFormNetworkCrawl,
});
...@@ -6,6 +6,7 @@ SimpleSchema.extendOptions(['autoform']); ...@@ -6,6 +6,7 @@ SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({ export default new SimpleSchema({
pathDirectory: { pathDirectory: {
type: String, type: String,
required: false,
label: 'Url du chemin :', label: 'Url du chemin :',
}, },
fileType: { fileType: {
...@@ -25,5 +26,20 @@ export default new SimpleSchema({ ...@@ -25,5 +26,20 @@ export default new SimpleSchema({
'fileType.$': { 'fileType.$': {
type: String, type: String,
}, },
uploadFile: {
type: String,
label: 'Upload de fichier :',
autoform: {
type: 'file',
webkitdirectory: 'webkitdirectory',
directory: 'directory',
multiple: 'multiple',
},
},
listFile: {
type: Object,
blackbox: true,
required: false,
},
}, { tracker: Tracker }); }, { tracker: Tracker });
import { FilesCollection } from 'meteor/ostrio:files';
import { Mongo } from 'meteor/mongo';
/**
* @type {Mongo.Collection}
*/
const TreeData = new Mongo.Collection('TreeData');
const Files = new FilesCollection({
collectionName: 'Files',
allowClientCode: false, // Disallow remove files from Client
onBeforeUpload(file) {
console.log(file.allowedExtensions, file.extension, new RegExp(file.allowedExtensions, 'i').test(file.extension));
if (file.size > 10485760) {
return `(${file.name}) Please upload file with size equal or less than 10MB`;
}
return new RegExp(file.allowedExtensions, 'i').test(file.extension);
},
});
export { TreeData, Files };
import { Mongo } from 'meteor/mongo';
/**
* this collection is local
* @type {Mongo.Collection}
*/
export default new Mongo.Collection('TreeData');
...@@ -2,15 +2,12 @@ ...@@ -2,15 +2,12 @@
import { check } from 'meteor/check'; import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema'; import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import formReseauCrawlSchema from '/imports/api/crawl/reseau/formReseauCrawlSchema'; import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric'; import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite'; import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi'; import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexReseau from '/imports/api/indexation/server/indexReseau'; import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import { addNotif } from '/imports/api/notif/methods'; import { addNotif } from '/imports/api/notif/methods';
import TreeData from '/imports/api/crawl/reseau/reseauCollection';
import dirTree from 'directory-tree';
import _ from 'lodash';
Meteor.methods({ Meteor.methods({
initIndexElastic() { initIndexElastic() {
...@@ -120,46 +117,17 @@ Meteor.methods({ ...@@ -120,46 +117,17 @@ Meteor.methods({
}); });
} }
}, },
indexReseau(data) { // display all file in network
crawlNetwork(data) {
check(data, Object); check(data, Object);
formReseauCrawlSchema.validate(data); // formNetworkCrawlSchema.validate(data);
this.unblock(); this.unblock();
const indexNetwork = new IndexNetwork();
const extensions = data.fileType ? data.fileType.join('|') : ''; indexNetwork.crawlNetwork(data);
const regexExtensions = new RegExp(`(${extensions})`); },
indexNetwork(files) {
const filteredTree = dirTree(data.pathDirectory, { extensions: regexExtensions }); check(files, Array);
const indexNetwork = new IndexNetwork();
console.log(filteredTree); indexNetwork.initParseFiles(files);
function insertTestData(parent, children) {
_.forEach(children, (item) => {
const idParent = TreeData.insert({
name: item.name,
parent,
type: item.type,
path: item.path,
extension: item.extension,
});
if (item.children && item.children.length) {
insertTestData(idParent, item.children);
}
});
}
TreeData.remove({});
if (filteredTree) {
const idParent = TreeData.insert({
name: filteredTree.name,
parent: null,
type: filteredTree.type,
path: filteredTree.path,
extension: null,
});
insertTestData(idParent, filteredTree.children);
} else {
throw new Meteor.Error('Chemin invalide!');
}
}, },
}); });
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import TreeData from '/imports/api/crawl/reseau/reseauCollection'; import { Files, TreeData } from '/imports/api/crawl/network/networkCollection';
Meteor.publish('TreeData', () => TreeData.find()); Meteor.publish('TreeData', () => TreeData.find({ userId: Meteor.userId() }));
Meteor.publish('files.images.all', () => Files.find().cursor);
import fs from 'fs';
import { TreeData } from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
import _ from 'lodash';
import IndexGeneric from './indexGeneric';
export default class IndexNetwork extends IndexGeneric {
crawlNetwork(files) {
console.log(files);
/* this.listNetwork = [];
_.forEach(files, (file) => {
const extension = file.name.split('.').pop();
console.log(extension);
});*/
/* const extensions = data.fileType ? data.fileType.join('|') : '';
const regexExtensions = new RegExp(`(${extensions})`);
// filter by extensions
const filteredTree = dirTree(data.pathDirectory, { extensions: regexExtensions });
// insert file recursively
function insertTestData(parent, children) {
_.remove(children, n => n.children && n.children.length === 0);
_.forEach(children, (item) => {
if (item.children && item.children.length) {
_.remove(item.children, n => n.extension && !regexExtensions.test(n.extension));
_.remove(item, n => n.children && n.children.length === 0);
}
const idParent = TreeData.insert({
name: item.name,
parent,
type: item.type,
path: item.path,
extension: item.extension,
userId: Meteor.userId(),
});
if (item.children && item.children.length) {
insertTestData(idParent, item.children);
}
});
_.remove(children, n => n.children && n.children.length === 0);
}
TreeData.remove({});
if (filteredTree && filteredTree.children && filteredTree.children.length) {
const idParent = TreeData.insert({
name: filteredTree.name,
parent: null,
type: filteredTree.type,
path: filteredTree.path,
extension: null,
userId: Meteor.userId(),
});
insertTestData(idParent, filteredTree.children);
} else {
throw new Meteor.Error('Chemin invalide!');
} */
}
initParseFiles(files) {
/* await Promise.all(files.map(async (file) => {
const contents = await fs.readFile(file.path, 'utf8');
console.log('sync '+fs.readFileSync(file.path, 'utf8'));
console.log('notsync '+contents);
}));*/
_.forEach(files, (file) => {
switch (file.extension) {
case 'json':
this.parseJsonFile(file);
break;
default:
break;
}
});
}
parseJsonFile(file) {
console.log(JSON.parse(fs.readFileSync(file.path, 'utf8')));
}
}
import fs from 'fs';
import dirTree from 'directory-tree';
import IndexGeneric from './indexGeneric';
export default class IndexReseau extends IndexGeneric {
initIndexation() {
// console.log(this.walkSync('/home/ngoura/', []));
/* const filteredTree = dirTree('/home/ngoura/Documents', { extensions: /\.pdf/ }, (item, PATH) => {
console.log(item);
});*/
}
walkSync(dir, filelist) {
const files = fs.readdirSync(dir);
filelist = filelist || [];
files.forEach((file) => {
// ne pas parcourir les fichiers cachés
if (file[0] !== '.') {
if (fs.statSync(dir + file).isDirectory()) {
filelist = this.walkSync(`${dir + file}/`, filelist);
} else {
filelist.push(file);
}
}
});
return filelist;
}
}
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
import $ from 'jquery'; import $ from 'jquery';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import '/imports/api/crawl/api/formApiCrawlHooks'; import '/imports/api/crawl/api/formApiCrawlHooks';
......
...@@ -2,13 +2,13 @@ ...@@ -2,13 +2,13 @@
import { Meteor } from 'meteor/meteor'; import { Meteor } from 'meteor/meteor';
import { Template } from 'meteor/templating'; import { Template } from 'meteor/templating';
import { Session } from 'meteor/session'; import { Session } from 'meteor/session';
import TabsCollection from '../../../../api/tabs/tabsCollection'; import TabsCollection from '/imports/api/tabs/tabsCollection';
import '../../../components/tabs/tabs'; import '/imports/ui/components/tabs/tabs';
import './main/main'; import './main/main';
import './site/site'; import './site/site';
import './api/api'; import './api/api';
import './reseau/reseau'; import './network/network';
import './indexation.html'; import './indexation.html';
......
<template name="networkIndexationTpl">
<div class="panel panel-default wrapper">
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des réseaux</h4>
<!-- webkitdirectory directory multiple-->
<!-- <form id="uploadFileNetwork" enctype="multipart/form-data" method="post">
<input id="inputDirectoryNetwork" name="uploadFileNetwork" type="file" webkitdirectory directory multiple/>
<input type="submit" value="upload" />
</form> -->
{{#each listAlertNetwork}}
<div class="alert alert-dismissable {{type}}">
<a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
<strong>{{title}} : </strong> {{message}}
</div>
{{/each}}
{{#autoForm id="formNetworkCrawl" schema=formNetworkCrawlSchema method="post" enctype="multipart/form-data"}}
<fieldset>
<div class="form-group">
{{#with currentUpload}}
Uploading <b>{{file.name}}</b>:
<span id="progress">{{progress.get}}%</span>
{{else}}
{{> afQuickField name='uploadFile' id='uploadFile' }}
{{/with}}
</div>
{{> afQuickField name='fileType'}}
</fieldset>
<button type="submit" class="btn btn-primary">Submit</button>
{{/autoForm}}
<hr>
<h4 class="text-center">Arborescence</h4>
{{#if Template.subscriptionsReady}}
{{> TreeView treeArgs }}
{{/if}}
<button id="buttonIndexNetwork" class="btn btn-success">Indexer</button>
</div>
</div>
</template>
import { Meteor } from 'meteor/meteor';
import { ReactiveVar } from 'meteor/reactive-var';
import { Template } from 'meteor/templating';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection';
import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import '/imports/api/crawl/network/formNetworkCrawlHooks';
import './network.html';
const listAlertNetwork = [];
const listAllFile = [];
const listFileForIndex = [];
Template.networkIndexationTpl.onCreated(function () {
this.currentUpload = new ReactiveVar(false);
this.listAlertNetwork = new ReactiveVar([]);
// Meteor.subscribe('files.images.all');
console.log(this);
});
Template.networkIndexationTpl.helpers({
currentUpload() {
return Template.instance().currentUpload.get();
},
listAlertNetwork() {
return Template.instance().listAlertNetwork.get();
},
setCurrentUpload(newData) {
Template.instance().currentUpload.set(newData);
},
setListAlertNetwork(newData) {
Template.instance().listAlertNetwork.set(newData);
},
});
Template.networkIndexationTpl.events({
'click #buttonIndexNetwork': () => {
console.log(listFileForIndex);
Meteor.call('indexNetwork', listFileForIndex);
},
});
Template.networkIndexationTpl.helpers({
treeArgs: {
collection: TreeData,
subscription: 'TreeData',
parent: null,
select: '',
openAll: false,
selectAll: false,
mapping: {
text: 'name',
extension: 'extension',
path: 'path',
aAttr(item) {
listAllFile[item._id] = item;
},
},
jstree: {
plugins: [
'checkbox',
'sort',
'state',
],
},
events: {
changed(e, listId, data) {
listId.forEach((id) => {
if (listAllFile[id].type === 'file' && listAllFile[id].extension) {
listFileForIndex.push(listAllFile[id]);
}
});
},
},
},
formNetworkCrawlSchema() {
return formNetworkCrawlSchema;
},
});
<template name="reseauIndexationTpl">
<div class="panel panel-default wrapper">
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des réseaux</h4>
{{> quickForm id="formReseauCrawl" schema=formReseauCrawlSchema type="insert" type="method" meteormethod="indexReseau"}}
<hr>
<h4 class="text-center">Arborescence</h4>
{{> TreeView treeArgs }}
</div>
</div>
</template>
import { Meteor } from 'meteor/meteor';
import { Template } from 'meteor/templating';
import TreeData from '/imports/api/crawl/reseau/reseauCollection';
import formReseauCrawlSchema from '/imports/api/crawl/reseau/formReseauCrawlSchema';
import './reseau.html';
Template.reseauIndexationTpl.events({
});
Template.reseauIndexationTpl.helpers({
treeArgs: {
collection: TreeData,
subscription: 'TreeData',
parent: null,
select: '',
openAll: true,
selectAll: false,
mapping: {
text: 'name',
aAttr(item) {},
},
jstree: {
plugins: [
'checkbox',
'sort',
'state',
],
},
events: {
changed(e, item, data) {},
},
},
formReseauCrawlSchema() {
return formReseauCrawlSchema;
},
});
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
}, },
{ {
"module": "indexation", "module": "indexation",
"layout": "reseauIndexationTpl", "layout": "networkIndexationTpl",
"label": "index Réseau", "label": "index Réseau",
"state": {}, "state": {},
"activ": false, "activ": false,
......
...@@ -1052,13 +1052,12 @@ eslint-import-resolver-meteor@^0.4.0: ...@@ -1052,13 +1052,12 @@ eslint-import-resolver-meteor@^0.4.0:
object-assign "^4.0.1" object-assign "^4.0.1"
resolve "^1.1.6" resolve "^1.1.6"
eslint-import-resolver-node@^0.2.0: eslint-import-resolver-node@^0.3.1:
version "0.2.3" version "0.3.1"
resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.2.3.tgz#5add8106e8c928db2cba232bcd9efa846e3da16c" resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.1.tgz#4422574cde66a9a7b099938ee4d508a199e0e3cc"
dependencies: dependencies:
debug "^2.2.0" debug "^2.6.8"