Commit 11d203aa authored by Nacim Goura's avatar Nacim Goura

add crawl twitter

parent 46217035
...@@ -46,3 +46,4 @@ deanius:promise ...@@ -46,3 +46,4 @@ deanius:promise
dynamic-import dynamic-import
aldeed:tabular aldeed:tabular
ajduke:bootstrap-tagsinput ajduke:bootstrap-tagsinput
percolate:synced-cron
...@@ -7,7 +7,7 @@ aldeed:collection2-core@2.0.1 ...@@ -7,7 +7,7 @@ aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1 aldeed:tabular@2.1.1
allow-deny@1.0.5 allow-deny@1.0.5
autoupdate@1.3.12 autoupdate@1.3.12
babel-compiler@6.19.2 babel-compiler@6.19.3
babel-runtime@1.0.1 babel-runtime@1.0.1
base64@1.0.10 base64@1.0.10
binary-heap@1.0.10 binary-heap@1.0.10
...@@ -31,7 +31,7 @@ deanius:promise@3.1.3 ...@@ -31,7 +31,7 @@ deanius:promise@3.1.3
deps@1.0.12 deps@1.0.12
diff-sequence@1.0.7 diff-sequence@1.0.7
dynamic-import@0.1.1 dynamic-import@0.1.1
ecmascript@0.8.0 ecmascript@0.8.1
ecmascript-runtime@0.4.1 ecmascript-runtime@0.4.1
ecmascript-runtime-client@0.4.2 ecmascript-runtime-client@0.4.2
ecmascript-runtime-server@0.4.1 ecmascript-runtime-server@0.4.1
...@@ -62,7 +62,7 @@ minifier-js@2.1.0 ...@@ -62,7 +62,7 @@ minifier-js@2.1.0
minimongo@1.2.1 minimongo@1.2.1
mobile-experience@1.0.4 mobile-experience@1.0.4
mobile-status-bar@1.0.14 mobile-status-bar@1.0.14
modules@0.9.1 modules@0.9.2
modules-runtime@0.8.0 modules-runtime@0.8.0
momentjs:moment@2.18.1 momentjs:moment@2.18.1
mongo@1.1.18 mongo@1.1.18
...@@ -71,6 +71,7 @@ npm-bcrypt@0.9.3 ...@@ -71,6 +71,7 @@ npm-bcrypt@0.9.3
npm-mongo@2.2.24 npm-mongo@2.2.24
observe-sequence@1.0.16 observe-sequence@1.0.16
ordered-dict@1.0.9 ordered-dict@1.0.9
percolate:synced-cron@1.3.2
practicalmeteor:chai@2.1.0_1 practicalmeteor:chai@2.1.0_1
practicalmeteor:loglevel@1.2.0_2 practicalmeteor:loglevel@1.2.0_2
practicalmeteor:mocha@2.4.5_6 practicalmeteor:mocha@2.4.5_6
......
,ngoura,ngoura,15.06.2017 09:13,file:///home/ngoura/.config/libreoffice/4;
\ No newline at end of file
import CrawlFacebook from './crawlFacebook'; import CrawlFacebook from './crawlFacebook';
import CrawlTwitter from './crawlTwitter';
export default class crawlWebsite { export default class crawlWebsite {
constructor(data) { constructor(config) {
switch (data.type) { switch (config.type) {
case 'facebook': case 'facebook':
// facebook; return new CrawlFacebook(config);
return new CrawlFacebook(data);
case 'twitter': case 'twitter':
return new CrawlTwitter(config);
break;
case 'instagram': case 'instagram':
break; break;
......
...@@ -5,12 +5,13 @@ import checkData from '../../utils/checkData'; ...@@ -5,12 +5,13 @@ import checkData from '../../utils/checkData';
export default class CrawlFacebook { export default class CrawlFacebook {
constructor(data) { constructor(config) {
this.config = config;
console.log('crawl Facebook'); console.log('crawl Facebook');
this.listDataForIndex = []; this.listDataForIndex = [];
if (data.content) { if (config.content) {
this.content = JSON.parse(data.content); this.content = JSON.parse(config.content);
return this.start(); return this.start();
} }
throw new Meteor.Error('Error', 'aucune donnée Facebook!'); throw new Meteor.Error('Error', 'aucune donnée Facebook!');
...@@ -33,6 +34,7 @@ export default class CrawlFacebook { ...@@ -33,6 +34,7 @@ export default class CrawlFacebook {
_.forEach(this.content.data, (item, index) => { _.forEach(this.content.data, (item, index) => {
const dataForIndex = { const dataForIndex = {
tag: 'social', tag: 'social',
domain: this.config.idPage,
url: `https://www.facebook.com/${item.id}`, url: `https://www.facebook.com/${item.id}`,
description: '', description: '',
body: '', body: '',
......
import { Meteor } from 'meteor/meteor';
import Twitter from 'twitter';
import _ from 'lodash';
import checkData from '../../utils/checkData';
export default class CrawlTwitter {
constructor(config) {
console.log('crawl Twittter');
this.config = config;
this.listDataForIndex = [];
this.client = new Twitter({
consumer_key: 'queq2peGONZ4NQxHLP1VAzTtx',
consumer_secret: '08EDBSNuDkg4OhZfuxJCq08g3OSrhGLETncW43NYHtK7AJggaY',
access_token_key: '40486089-1raVGT0dF3pV5eobkuDGPvlPy8frDckXaIh3AlY0e',
access_token_secret: 'JJrJ3UT0B8Xjc9mnKqDXnGTNp7hKYmtcurNhfK1KVT9Bm',
});
return this.start();
}
/**
*
* @returns Promise
*/
async start() {
try {
const tweets = await this.client.get('statuses/user_timeline', { screen_name: this.config.idPage });
return this.parseData(tweets);
} catch (e) {
throw new Meteor.Error('Error', 'Erreur lors l\'utilisation de l\'API Twitter', 'statuses/user_timeline');
}
}
/**
*
* @returns Promise
*/
parseData(tweets) {
return new Promise((resolve) => {
_.forEach(tweets, (item, index) => {
const dataForIndex = {
tag: 'social',
domain: this.config.idPage,
url: `https://twitter.com/${this.config.idPage}/status/${item.id_str}`,
};
dataForIndex.urlText = checkData.slugText(checkData.cleanText(dataForIndex.url));
if (item.text) {
item.text = item.text.replace(/^(?:https?:)?\/\//, '');
dataForIndex.title = checkData.cleanText(item.text);
dataForIndex.title_suggest = checkData.cleanText(item.text);
dataForIndex.description = checkData.cleanText(item.text);
dataForIndex.body = checkData.cleanText(item.text);
}
if (item.created_time) {
dataForIndex.createdAt = item.created_at;
}
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType,
_id: dataForIndex.url,
},
});
this.listDataForIndex.push(dataForIndex);
if (index + 1 === tweets.length) {
resolve(this.listDataForIndex);
}
});
});
}
}
...@@ -20,7 +20,6 @@ export default class crawlWebsite { ...@@ -20,7 +20,6 @@ export default class crawlWebsite {
}; };
try { try {
const syncFunc = Meteor.wrapAsync(Meteor.call); const syncFunc = Meteor.wrapAsync(Meteor.call);
this.config = syncFunc('getConfig'); this.config = syncFunc('getConfig');
} catch (err) { } catch (err) {
console.error(err); console.error(err);
...@@ -122,6 +121,7 @@ export default class crawlWebsite { ...@@ -122,6 +121,7 @@ export default class crawlWebsite {
const title = checkData.cleanText($('title').text()); const title = checkData.cleanText($('title').text());
const dataForIndex = { const dataForIndex = {
tag: 'site', tag: 'site',
domain: this.config.domain,
title, title,
title_suggest: { title_suggest: {
input: title, input: title,
......
...@@ -86,6 +86,9 @@ exports.mapping = { ...@@ -86,6 +86,9 @@ exports.mapping = {
tag: { tag: {
type: 'keyword', type: 'keyword',
}, },
domain: {
type: 'keyword'
},
title: { title: {
type: 'text', type: 'text',
analyzer: 'french_light', analyzer: 'french_light',
......
...@@ -4,8 +4,9 @@ import SimpleSchema from 'simpl-schema'; ...@@ -4,8 +4,9 @@ import SimpleSchema from 'simpl-schema';
SimpleSchema.extendOptions(['autoform']); SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({ export default new SimpleSchema({
AccessToken: { accessToken: {
type: String, type: String,
required: false,
autoform: { autoform: {
type: 'hidden', type: 'hidden',
label: false, label: false,
......
...@@ -6,7 +6,7 @@ export default { ...@@ -6,7 +6,7 @@ export default {
displayNotif({ displayNotif({
type: 'success', type: 'success',
title: 'Indexation : ', title: 'Indexation : ',
message: 'Indexation fini avec succès!', message: `Indexation fini avec succès avec ${result.items.length} urls indexées!`,
save: true, save: true,
}); });
}, },
......
...@@ -6,14 +6,19 @@ ...@@ -6,14 +6,19 @@
<div class="panel-body"> <div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des API</h4> <h4 class="text-center">Gestion de l'indexation des API</h4>
<hr> {{#autoForm id="formApiCrawl" schema=formApiCrawlSchema buttonContent="Valider" type="method" meteormethod="indexApi" resetOnSuccess=false}}
<p>Pour pouvoir utiliser les API, il faut d'abord se connecter afin d'avoir les autorisations nécessaires!</p> <fieldset>
<p class="text-danger">L'id de la page doit être préalablement indiqué!</p> {{> afQuickField name='idPage'}}
<button id="loginButtonFacebook" class="btn btn-success">Login facebook</button> {{> afQuickField name='accessToken'}}
<hr> {{> afQuickField name='type'}}
<div class="hidden-facebook hidden">
{{> quickForm id="formApiCrawl" schema=formApiCrawlSchema buttonContent="Valider" type="method" meteormethod="indexApi" }} <p class="text-danger">Une connexion est obligatoire!</p>
<button id="loginButtonFacebook" class="btn btn-success">Login facebook</button>
</div>
{{> afQuickField name='content'}}
</fieldset>
<button type="submit" class="btn btn-primary">Insert</button>
{{/autoForm}}
</div> </div>
</div> </div>
......
...@@ -21,7 +21,7 @@ Template.apiIndexationTpl.events({ ...@@ -21,7 +21,7 @@ Template.apiIndexationTpl.events({
if (inputIdPage.length > 0) { if (inputIdPage.length > 0) {
FB.login(() => { FB.login(() => {
FB.api(`${$('input[name=idPage]').val()}?fields=access_token`, (res) => { FB.api(`${$('input[name=idPage]').val()}?fields=access_token`, (res) => {
$('input[name=AccessToken]').val(res.access_token); $('input[name=accessToken]').val(res.access_token);
FB.api('me/feed', { access_token: res.access_token }, (result) => { FB.api('me/feed', { access_token: res.access_token }, (result) => {
$('input[name=content]').val(JSON.stringify(result)); $('input[name=content]').val(JSON.stringify(result));
}); });
...@@ -31,6 +31,14 @@ Template.apiIndexationTpl.events({ ...@@ -31,6 +31,14 @@ Template.apiIndexationTpl.events({
alert('L\'ID de la page n\'est pas indiqué'); alert('L\'ID de la page n\'est pas indiqué');
} }
}, },
'click input[name=type]': (event) => {
const value = $(event.target).val();
if (value === 'facebook') {
$('.hidden-facebook').removeClass('hidden');
} else {
$('.hidden-facebook').addClass('hidden');
}
},
}); });
Template.apiIndexationTpl.helpers({ Template.apiIndexationTpl.helpers({
......
...@@ -37,12 +37,13 @@ Template.siteIndexationTpl.events({ ...@@ -37,12 +37,13 @@ Template.siteIndexationTpl.events({
$(event.target).find('h4').removeClass('hidden'); $(event.target).find('h4').removeClass('hidden');
Meteor.callPromise('indexWebsite', url) Meteor.callPromise('indexWebsite', url)
.then(() => { .then((result) => {
console.log(result);
$(event.target).find('h4').addClass('hidden'); $(event.target).find('h4').addClass('hidden');
displayNotif({ displayNotif({
type: 'success', type: 'success',
title: 'Indexation : ', title: 'Indexation : ',
message: 'Indexation fini avec succès!', message: `Indexation de ${url} fini avec succès avec ${result.items.length} pages indexées!`,
save: true, save: true,
}); });
}).catch((error) => { }).catch((error) => {
...@@ -55,7 +56,6 @@ Template.siteIndexationTpl.events({ ...@@ -55,7 +56,6 @@ Template.siteIndexationTpl.events({
save: true, save: true,
}); });
}); });
event.target.reset();
}, },
'submit #formReglageSite': (event) => { 'submit #formReglageSite': (event) => {
event.preventDefault(); event.preventDefault();
......
...@@ -22,7 +22,8 @@ ...@@ -22,7 +22,8 @@
"simpl-schema": "^0.3.1", "simpl-schema": "^0.3.1",
"sitemapper": "^2.1.13", "sitemapper": "^2.1.13",
"slug": "^0.9.1", "slug": "^0.9.1",
"sweetalert2": "^6.6.5" "sweetalert2": "^6.6.5",
"twitter": "^1.7.1"
}, },
"devDependencies": { "devDependencies": {
"@meteorjs/eslint-config-meteor": "^1.0.5", "@meteorjs/eslint-config-meteor": "^1.0.5",
......
...@@ -784,6 +784,10 @@ deep-extend@^0.4.1, deep-extend@~0.4.0: ...@@ -784,6 +784,10 @@ deep-extend@^0.4.1, deep-extend@~0.4.0:
version "0.4.1" version "0.4.1"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253"
deep-extend@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.0.tgz#6ef4a09b05f98b0e358d6d93d4ca3caec6672803"
deep-is@~0.1.3: deep-is@~0.1.3:
version "0.1.3" version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
...@@ -2393,7 +2397,7 @@ request-promise@^4.1.0: ...@@ -2393,7 +2397,7 @@ request-promise@^4.1.0:
stealthy-require "^1.1.0" stealthy-require "^1.1.0"
tough-cookie ">=2.3.0" tough-cookie ">=2.3.0"
request@^2.74.0, request@^2.79.0, request@~2.79.0: request@^2.72.0, request@^2.74.0, request@^2.79.0, request@~2.79.0:
version "2.79.0" version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies: dependencies:
...@@ -2762,6 +2766,13 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0: ...@@ -2762,6 +2766,13 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0:
version "0.14.5" version "0.14.5"
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
twitter@^1.7.1:
version "1.7.1"
resolved "https://registry.yarnpkg.com/twitter/-/twitter-1.7.1.tgz#0762378f1dc1c050e48f666aca904e24b1a962f4"
dependencies:
deep-extend "^0.5.0"
request "^2.72.0"
type-check@~0.3.2: type-check@~0.3.2:
version "0.3.2" version "0.3.2"
resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment