Commit 8a4d0043 authored by Nacim Goura's avatar Nacim Goura

update crawl network

parent e0544ebf
......@@ -11,6 +11,7 @@ mongo@1.1.19-rc.1 # The database Meteor supports right now
blaze-html-templates # Compile .html files into Meteor Blaze views
reactive-var@1.0.11 # Reactive variable for tracker
tracker@1.1.3 # Meteor's client-side reactive programming library
dynamic-import@0.1.1
# package for compile and minify
standard-minifier-css@1.3.4 # CSS minifier run for production mode
......@@ -23,29 +24,30 @@ fortawesome:fontawesome # pretty icon
# package for view
kadira:flow-router # FlowRouter is a very simple router for Meteor
kadira:blaze-layout # Layout manager for blaze (works well with FlowRouter)
ajduke:bootstrap-tagsinput # tag input
# package for test
# package for test or validate
practicalmeteor:mocha # A package for writing and running your meteor app and package tests with mocha
johanbrook:publication-collector # Test a Meteor publication by collecting its output
practicalmeteor:chai
check@1.2.5
# package for data
session@1.1.7
aldeed:collection2-core
dburles:collection-helpers
aldeed:autoform
check@1.2.5
matb33:collection-hooks
# package for user
accounts-base@1.3.1-rc.1
accounts-password@1.4.0-rc.1
alanning:roles
practicalmeteor:chai
# other packages
deanius:promise
dynamic-import@0.1.1
aldeed:tabular
ajduke:bootstrap-tagsinput
percolate:synced-cron
matb33:collection-hooks
qnipp:treeview
ostrio:files
aldeed:template-extension
......@@ -5,6 +5,7 @@ alanning:roles@1.2.16
aldeed:autoform@6.2.0
aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1
aldeed:template-extension@4.1.0
allow-deny@1.0.6-rc.1
autoupdate@1.3.12
babel-compiler@6.19.3
......@@ -72,6 +73,8 @@ npm-bcrypt@0.9.3
npm-mongo@2.2.24
observe-sequence@1.0.16
ordered-dict@1.0.9
ostrio:cookies@2.2.1
ostrio:files@1.8.0
percolate:synced-cron@1.3.2
practicalmeteor:chai@2.1.0_1
practicalmeteor:loglevel@1.2.0_2
......
import { AutoForm } from 'meteor/aldeed:autoform';
import { _ } from 'meteor/underscore';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
const hooksFormNetworkCrawl = {
onSubmit: function (insertDoc, updateDoc, currentDoc) {
const templateInstance = this.template.parent();
const listAlertNetwork = [];
if (insertDoc) {
const files = this.event.currentTarget.uploadFile.files;
const extensions = insertDoc.fileType ? insertDoc.fileType.join('|') : '';
if (files && extensions) {
console.log(extensions);
_.each(files, function (file) {
file.allowedExtensions = extensions;
const upload = Files.insert({
file,
streams: 'dynamic',
chunkSize: 'dynamic',
}, false);
upload.on('start', function () {
console.log('start upload!');
templateInstance.currentUpload.set(this);
});
upload.on('end', (error, fileObj) => {
console.log('end upload');
if (error) {
listAlertNetwork.push({
type: 'alert-danger',
title: 'Erreur',
message: `${fileObj.name} ${error}`,
});
} else {
listAlertNetwork.push({
type: 'alert-success',
title: 'Succès',
message: `File "${fileObj.name}" successfully uploaded`,
});
}
templateInstance.listAlertNetwork.set(listAlertNetwork);
templateInstance.currentUpload.set(false);
});
upload.start();
});
}
}
return false;
},
onSuccess(formType, result) {},
onError(formType, error) {},
};
AutoForm.hooks({
formNetworkCrawl: hooksFormNetworkCrawl,
});
......@@ -6,6 +6,7 @@ SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
pathDirectory: {
type: String,
required: false,
label: 'Url du chemin :',
},
fileType: {
......@@ -25,5 +26,20 @@ export default new SimpleSchema({
'fileType.$': {
type: String,
},
uploadFile: {
type: String,
label: 'Upload de fichier :',
autoform: {
type: 'file',
webkitdirectory: 'webkitdirectory',
directory: 'directory',
multiple: 'multiple',
},
},
listFile: {
type: Object,
blackbox: true,
required: false,
},
}, { tracker: Tracker });
import { FilesCollection } from 'meteor/ostrio:files';
import { Mongo } from 'meteor/mongo';
/**
* @type {Mongo.Collection}
*/
const TreeData = new Mongo.Collection('TreeData');
const Files = new FilesCollection({
collectionName: 'Files',
allowClientCode: false, // Disallow remove files from Client
onBeforeUpload(file) {
console.log(file.allowedExtensions, file.extension, new RegExp(file.allowedExtensions, 'i').test(file.extension));
if (file.size > 10485760) {
return `(${file.name}) Please upload file with size equal or less than 10MB`;
}
return new RegExp(file.allowedExtensions, 'i').test(file.extension);
},
});
export { TreeData, Files };
import { Mongo } from 'meteor/mongo';
/**
* this collection is local
* @type {Mongo.Collection}
*/
export default new Mongo.Collection('TreeData');
......@@ -2,15 +2,12 @@
import { check } from 'meteor/check';
import { Meteor } from 'meteor/meteor';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import formReseauCrawlSchema from '/imports/api/crawl/reseau/formReseauCrawlSchema';
import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import IndexGeneric from '/imports/api/indexation/server/indexGeneric';
import IndexWebsite from '/imports/api/indexation/server/indexWebsite';
import IndexApi from '/imports/api/indexation/server/indexApi';
import IndexReseau from '/imports/api/indexation/server/indexReseau';
import IndexNetwork from '/imports/api/indexation/server/indexNetwork';
import { addNotif } from '/imports/api/notif/methods';
import TreeData from '/imports/api/crawl/reseau/reseauCollection';
import dirTree from 'directory-tree';
import _ from 'lodash';
Meteor.methods({
initIndexElastic() {
......@@ -120,46 +117,17 @@ Meteor.methods({
});
}
},
indexReseau(data) {
// display all file in network
crawlNetwork(data) {
check(data, Object);
formReseauCrawlSchema.validate(data);
// formNetworkCrawlSchema.validate(data);
this.unblock();
const extensions = data.fileType ? data.fileType.join('|') : '';
const regexExtensions = new RegExp(`(${extensions})`);
const filteredTree = dirTree(data.pathDirectory, { extensions: regexExtensions });
console.log(filteredTree);
function insertTestData(parent, children) {
_.forEach(children, (item) => {
const idParent = TreeData.insert({
name: item.name,
parent,
type: item.type,
path: item.path,
extension: item.extension,
});
if (item.children && item.children.length) {
insertTestData(idParent, item.children);
}
});
}
TreeData.remove({});
if (filteredTree) {
const idParent = TreeData.insert({
name: filteredTree.name,
parent: null,
type: filteredTree.type,
path: filteredTree.path,
extension: null,
});
insertTestData(idParent, filteredTree.children);
} else {
throw new Meteor.Error('Chemin invalide!');
}
const indexNetwork = new IndexNetwork();
indexNetwork.crawlNetwork(data);
},
indexNetwork(files) {
check(files, Array);
const indexNetwork = new IndexNetwork();
indexNetwork.initParseFiles(files);
},
});
import { Meteor } from 'meteor/meteor';
import TreeData from '/imports/api/crawl/reseau/reseauCollection';
import { Files, TreeData } from '/imports/api/crawl/network/networkCollection';
Meteor.publish('TreeData', () => TreeData.find());
Meteor.publish('TreeData', () => TreeData.find({ userId: Meteor.userId() }));
Meteor.publish('files.images.all', () => Files.find().cursor);
import fs from 'fs';
import { TreeData } from '/imports/api/crawl/network/networkCollection';
import dirTree from 'directory-tree';
import _ from 'lodash';
import IndexGeneric from './indexGeneric';
export default class IndexNetwork extends IndexGeneric {
crawlNetwork(files) {
console.log(files);
/* this.listNetwork = [];
_.forEach(files, (file) => {
const extension = file.name.split('.').pop();
console.log(extension);
});*/
/* const extensions = data.fileType ? data.fileType.join('|') : '';
const regexExtensions = new RegExp(`(${extensions})`);
// filter by extensions
const filteredTree = dirTree(data.pathDirectory, { extensions: regexExtensions });
// insert file recursively
function insertTestData(parent, children) {
_.remove(children, n => n.children && n.children.length === 0);
_.forEach(children, (item) => {
if (item.children && item.children.length) {
_.remove(item.children, n => n.extension && !regexExtensions.test(n.extension));
_.remove(item, n => n.children && n.children.length === 0);
}
const idParent = TreeData.insert({
name: item.name,
parent,
type: item.type,
path: item.path,
extension: item.extension,
userId: Meteor.userId(),
});
if (item.children && item.children.length) {
insertTestData(idParent, item.children);
}
});
_.remove(children, n => n.children && n.children.length === 0);
}
TreeData.remove({});
if (filteredTree && filteredTree.children && filteredTree.children.length) {
const idParent = TreeData.insert({
name: filteredTree.name,
parent: null,
type: filteredTree.type,
path: filteredTree.path,
extension: null,
userId: Meteor.userId(),
});
insertTestData(idParent, filteredTree.children);
} else {
throw new Meteor.Error('Chemin invalide!');
} */
}
initParseFiles(files) {
/* await Promise.all(files.map(async (file) => {
const contents = await fs.readFile(file.path, 'utf8');
console.log('sync '+fs.readFileSync(file.path, 'utf8'));
console.log('notsync '+contents);
}));*/
_.forEach(files, (file) => {
switch (file.extension) {
case 'json':
this.parseJsonFile(file);
break;
default:
break;
}
});
}
parseJsonFile(file) {
console.log(JSON.parse(fs.readFileSync(file.path, 'utf8')));
}
}
import fs from 'fs';
import dirTree from 'directory-tree';
import IndexGeneric from './indexGeneric';
export default class IndexReseau extends IndexGeneric {
initIndexation() {
// console.log(this.walkSync('/home/ngoura/', []));
/* const filteredTree = dirTree('/home/ngoura/Documents', { extensions: /\.pdf/ }, (item, PATH) => {
console.log(item);
});*/
}
walkSync(dir, filelist) {
const files = fs.readdirSync(dir);
filelist = filelist || [];
files.forEach((file) => {
// ne pas parcourir les fichiers cachés
if (file[0] !== '.') {
if (fs.statSync(dir + file).isDirectory()) {
filelist = this.walkSync(`${dir + file}/`, filelist);
} else {
filelist.push(file);
}
}
});
return filelist;
}
}
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import { Template } from 'meteor/templating';
import $ from 'jquery';
import formApiCrawlSchema from '/imports/api/crawl/api/formApiCrawlSchema';
import '/imports/api/crawl/api/formApiCrawlHooks';
......
......@@ -2,13 +2,13 @@
import { Meteor } from 'meteor/meteor';
import { Template } from 'meteor/templating';
import { Session } from 'meteor/session';
import TabsCollection from '../../../../api/tabs/tabsCollection';
import TabsCollection from '/imports/api/tabs/tabsCollection';
import '../../../components/tabs/tabs';
import '/imports/ui/components/tabs/tabs';
import './main/main';
import './site/site';
import './api/api';
import './reseau/reseau';
import './network/network';
import './indexation.html';
......
<template name="networkIndexationTpl">
<div class="panel panel-default wrapper">
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des réseaux</h4>
<!-- webkitdirectory directory multiple-->
<!-- <form id="uploadFileNetwork" enctype="multipart/form-data" method="post">
<input id="inputDirectoryNetwork" name="uploadFileNetwork" type="file" webkitdirectory directory multiple/>
<input type="submit" value="upload" />
</form> -->
{{#each listAlertNetwork}}
<div class="alert alert-dismissable {{type}}">
<a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
<strong>{{title}} : </strong> {{message}}
</div>
{{/each}}
{{#autoForm id="formNetworkCrawl" schema=formNetworkCrawlSchema method="post" enctype="multipart/form-data"}}
<fieldset>
<div class="form-group">
{{#with currentUpload}}
Uploading <b>{{file.name}}</b>:
<span id="progress">{{progress.get}}%</span>
{{else}}
{{> afQuickField name='uploadFile' id='uploadFile' }}
{{/with}}
</div>
{{> afQuickField name='fileType'}}
</fieldset>
<button type="submit" class="btn btn-primary">Submit</button>
{{/autoForm}}
<hr>
<h4 class="text-center">Arborescence</h4>
{{#if Template.subscriptionsReady}}
{{> TreeView treeArgs }}
{{/if}}
<button id="buttonIndexNetwork" class="btn btn-success">Indexer</button>
</div>
</div>
</template>
import { Meteor } from 'meteor/meteor';
import { ReactiveVar } from 'meteor/reactive-var';
import { Template } from 'meteor/templating';
import { TreeData, Files } from '/imports/api/crawl/network/networkCollection';
import formNetworkCrawlSchema from '/imports/api/crawl/network/formNetworkCrawlSchema';
import '/imports/api/crawl/network/formNetworkCrawlHooks';
import './network.html';
const listAlertNetwork = [];
const listAllFile = [];
const listFileForIndex = [];
Template.networkIndexationTpl.onCreated(function () {
this.currentUpload = new ReactiveVar(false);
this.listAlertNetwork = new ReactiveVar([]);
// Meteor.subscribe('files.images.all');
console.log(this);
});
Template.networkIndexationTpl.helpers({
currentUpload() {
return Template.instance().currentUpload.get();
},
listAlertNetwork() {
return Template.instance().listAlertNetwork.get();
},
setCurrentUpload(newData) {
Template.instance().currentUpload.set(newData);
},
setListAlertNetwork(newData) {
Template.instance().listAlertNetwork.set(newData);
},
});
Template.networkIndexationTpl.events({
'click #buttonIndexNetwork': () => {
console.log(listFileForIndex);
Meteor.call('indexNetwork', listFileForIndex);
},
});
Template.networkIndexationTpl.helpers({
treeArgs: {
collection: TreeData,
subscription: 'TreeData',
parent: null,
select: '',
openAll: false,
selectAll: false,
mapping: {
text: 'name',
extension: 'extension',
path: 'path',
aAttr(item) {
listAllFile[item._id] = item;
},
},
jstree: {
plugins: [
'checkbox',
'sort',
'state',
],
},
events: {
changed(e, listId, data) {
listId.forEach((id) => {
if (listAllFile[id].type === 'file' && listAllFile[id].extension) {
listFileForIndex.push(listAllFile[id]);
}
});
},
},
},
formNetworkCrawlSchema() {
return formNetworkCrawlSchema;
},
});
<template name="reseauIndexationTpl">
<div class="panel panel-default wrapper">
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des réseaux</h4>
{{> quickForm id="formReseauCrawl" schema=formReseauCrawlSchema type="insert" type="method" meteormethod="indexReseau"}}
<hr>
<h4 class="text-center">Arborescence</h4>
{{> TreeView treeArgs }}
</div>
</div>
</template>
import { Meteor } from 'meteor/meteor';
import { Template } from 'meteor/templating';
import TreeData from '/imports/api/crawl/reseau/reseauCollection';
import formReseauCrawlSchema from '/imports/api/crawl/reseau/formReseauCrawlSchema';
import './reseau.html';
Template.reseauIndexationTpl.events({
});
Template.reseauIndexationTpl.helpers({
treeArgs: {
collection: TreeData,
subscription: 'TreeData',
parent: null,
select: '',
openAll: true,
selectAll: false,
mapping: {
text: 'name',
aAttr(item) {},
},
jstree: {
plugins: [
'checkbox',
'sort',
'state',
],
},
events: {
changed(e, item, data) {},
},
},
formReseauCrawlSchema() {
return formReseauCrawlSchema;
},
});
......@@ -38,7 +38,7 @@
},
{
"module": "indexation",
"layout": "reseauIndexationTpl",
"layout": "networkIndexationTpl",
"label": "index Réseau",
"state": {},
"activ": false,
......
......@@ -1052,13 +1052,12 @@ eslint-import-resolver-meteor@^0.4.0:
object-assign "^4.0.1"
resolve "^1.1.6"
eslint-import-resolver-node@^0.2.0:
version "0.2.3"
resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.2.3.tgz#5add8106e8c928db2cba232bcd9efa846e3da16c"
eslint-import-resolver-node@^0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.1.tgz#4422574cde66a9a7b099938ee4d508a199e0e3cc"
dependencies:
debug "^2.2.0"
object-assign "^4.0.1"
resolve "^1.1.6"
debug "^2.6.8"
resolve "^1.2.0"
eslint-module-utils@^2.0.0:
version "2.0.0"
......@@ -1067,24 +1066,24 @@ eslint-module-utils@^2.0.0:
debug "2.2.0"
pkg-dir "^1.0.0"
eslint-plugin-import@^2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.6.0.tgz#2a4bbad36a078e052a3c830ce3dfbd6b8a12c6e5"
eslint-plugin-import@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.6.1.tgz#f580be62bb809421d46e338372764afcc9f59bf6"
dependencies:
builtin-modules "^1.1.1"
contains-path "^0.1.0"
debug "^2.6.8"
doctrine "1.5.0"
eslint-import-resolver-node "^0.2.0"
eslint-import-resolver-node "^0.3.1"
eslint-module-utils "^2.0.0"
has "^1.0.1"
lodash.cond "^4.3.0"
minimatch "^3.0.3"
read-pkg-up "^2.0.0"
eslint-plugin-jsx-a11y@^6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.0.0.tgz#d5cdd68f665a89540a55ddc0d898caf80ca94ac5"
eslint-plugin-jsx-a11y@^6.0.2:
version "6.0.2"
resolved "https://registry.yarnpkg.com/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.0.2.tgz#659277a758b036c305a7e4a13057c301cd3be73f"
dependencies:
aria-query "^0.7.0"
array-includes "^3.0.3"
......@@ -2441,7 +2440,7 @@ resolve-from@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226"
resolve@^1.1.6:
resolve@^1.1.6, resolve@^1.2.0:
version "1.3.3"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.3.3.tgz#655907c3469a8680dc2de3a275a8fdd69691f0e5"
dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment