Commit 11d203aa authored by Nacim Goura's avatar Nacim Goura

add crawl twitter

parent 46217035
......@@ -46,3 +46,4 @@ deanius:promise
dynamic-import
aldeed:tabular
ajduke:bootstrap-tagsinput
percolate:synced-cron
......@@ -7,7 +7,7 @@ aldeed:collection2-core@2.0.1
aldeed:tabular@2.1.1
allow-deny@1.0.5
autoupdate@1.3.12
babel-compiler@6.19.2
babel-compiler@6.19.3
babel-runtime@1.0.1
base64@1.0.10
binary-heap@1.0.10
......@@ -31,7 +31,7 @@ deanius:promise@3.1.3
deps@1.0.12
diff-sequence@1.0.7
dynamic-import@0.1.1
ecmascript@0.8.0
ecmascript@0.8.1
ecmascript-runtime@0.4.1
ecmascript-runtime-client@0.4.2
ecmascript-runtime-server@0.4.1
......@@ -62,7 +62,7 @@ minifier-js@2.1.0
minimongo@1.2.1
mobile-experience@1.0.4
mobile-status-bar@1.0.14
modules@0.9.1
modules@0.9.2
modules-runtime@0.8.0
momentjs:moment@2.18.1
mongo@1.1.18
......@@ -71,6 +71,7 @@ npm-bcrypt@0.9.3
npm-mongo@2.2.24
observe-sequence@1.0.16
ordered-dict@1.0.9
percolate:synced-cron@1.3.2
practicalmeteor:chai@2.1.0_1
practicalmeteor:loglevel@1.2.0_2
practicalmeteor:mocha@2.4.5_6
......
,ngoura,ngoura,15.06.2017 09:13,file:///home/ngoura/.config/libreoffice/4;
\ No newline at end of file
import CrawlFacebook from './crawlFacebook';
import CrawlTwitter from './crawlTwitter';
export default class crawlWebsite {
constructor(data) {
switch (data.type) {
constructor(config) {
switch (config.type) {
case 'facebook':
// facebook;
return new CrawlFacebook(data);
return new CrawlFacebook(config);
case 'twitter':
break;
return new CrawlTwitter(config);
case 'instagram':
break;
......
......@@ -5,12 +5,13 @@ import checkData from '../../utils/checkData';
export default class CrawlFacebook {
constructor(data) {
constructor(config) {
this.config = config;
console.log('crawl Facebook');
this.listDataForIndex = [];
if (data.content) {
this.content = JSON.parse(data.content);
if (config.content) {
this.content = JSON.parse(config.content);
return this.start();
}
throw new Meteor.Error('Error', 'aucune donnée Facebook!');
......@@ -33,6 +34,7 @@ export default class CrawlFacebook {
_.forEach(this.content.data, (item, index) => {
const dataForIndex = {
tag: 'social',
domain: this.config.idPage,
url: `https://www.facebook.com/${item.id}`,
description: '',
body: '',
......
import { Meteor } from 'meteor/meteor';
import Twitter from 'twitter';
import _ from 'lodash';
import checkData from '../../utils/checkData';
export default class CrawlTwitter {
constructor(config) {
console.log('crawl Twittter');
this.config = config;
this.listDataForIndex = [];
this.client = new Twitter({
consumer_key: 'queq2peGONZ4NQxHLP1VAzTtx',
consumer_secret: '08EDBSNuDkg4OhZfuxJCq08g3OSrhGLETncW43NYHtK7AJggaY',
access_token_key: '40486089-1raVGT0dF3pV5eobkuDGPvlPy8frDckXaIh3AlY0e',
access_token_secret: 'JJrJ3UT0B8Xjc9mnKqDXnGTNp7hKYmtcurNhfK1KVT9Bm',
});
return this.start();
}
/**
*
* @returns Promise
*/
async start() {
try {
const tweets = await this.client.get('statuses/user_timeline', { screen_name: this.config.idPage });
return this.parseData(tweets);
} catch (e) {
throw new Meteor.Error('Error', 'Erreur lors l\'utilisation de l\'API Twitter', 'statuses/user_timeline');
}
}
/**
*
* @returns Promise
*/
parseData(tweets) {
return new Promise((resolve) => {
_.forEach(tweets, (item, index) => {
const dataForIndex = {
tag: 'social',
domain: this.config.idPage,
url: `https://twitter.com/${this.config.idPage}/status/${item.id_str}`,
};
dataForIndex.urlText = checkData.slugText(checkData.cleanText(dataForIndex.url));
if (item.text) {
item.text = item.text.replace(/^(?:https?:)?\/\//, '');
dataForIndex.title = checkData.cleanText(item.text);
dataForIndex.title_suggest = checkData.cleanText(item.text);
dataForIndex.description = checkData.cleanText(item.text);
dataForIndex.body = checkData.cleanText(item.text);
}
if (item.created_time) {
dataForIndex.createdAt = item.created_at;
}
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
_type: Meteor.settings.private.elasticsearch.esType,
_id: dataForIndex.url,
},
});
this.listDataForIndex.push(dataForIndex);
if (index + 1 === tweets.length) {
resolve(this.listDataForIndex);
}
});
});
}
}
......@@ -20,7 +20,6 @@ export default class crawlWebsite {
};
try {
const syncFunc = Meteor.wrapAsync(Meteor.call);
this.config = syncFunc('getConfig');
} catch (err) {
console.error(err);
......@@ -122,6 +121,7 @@ export default class crawlWebsite {
const title = checkData.cleanText($('title').text());
const dataForIndex = {
tag: 'site',
domain: this.config.domain,
title,
title_suggest: {
input: title,
......
......@@ -86,6 +86,9 @@ exports.mapping = {
tag: {
type: 'keyword',
},
domain: {
type: 'keyword'
},
title: {
type: 'text',
analyzer: 'french_light',
......
......@@ -4,8 +4,9 @@ import SimpleSchema from 'simpl-schema';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
AccessToken: {
accessToken: {
type: String,
required: false,
autoform: {
type: 'hidden',
label: false,
......
......@@ -6,7 +6,7 @@ export default {
displayNotif({
type: 'success',
title: 'Indexation : ',
message: 'Indexation fini avec succès!',
message: `Indexation fini avec succès avec ${result.items.length} urls indexées!`,
save: true,
});
},
......
......@@ -6,14 +6,19 @@
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des API</h4>
<hr>
<p>Pour pouvoir utiliser les API, il faut d'abord se connecter afin d'avoir les autorisations nécessaires!</p>
<p class="text-danger">L'id de la page doit être préalablement indiqué!</p>
<button id="loginButtonFacebook" class="btn btn-success">Login facebook</button>
<hr>
{{> quickForm id="formApiCrawl" schema=formApiCrawlSchema buttonContent="Valider" type="method" meteormethod="indexApi" }}
{{#autoForm id="formApiCrawl" schema=formApiCrawlSchema buttonContent="Valider" type="method" meteormethod="indexApi" resetOnSuccess=false}}
<fieldset>
{{> afQuickField name='idPage'}}
{{> afQuickField name='accessToken'}}
{{> afQuickField name='type'}}
<div class="hidden-facebook hidden">
<p class="text-danger">Une connexion est obligatoire!</p>
<button id="loginButtonFacebook" class="btn btn-success">Login facebook</button>
</div>
{{> afQuickField name='content'}}
</fieldset>
<button type="submit" class="btn btn-primary">Insert</button>
{{/autoForm}}
</div>
</div>
......
......@@ -21,7 +21,7 @@ Template.apiIndexationTpl.events({
if (inputIdPage.length > 0) {
FB.login(() => {
FB.api(`${$('input[name=idPage]').val()}?fields=access_token`, (res) => {
$('input[name=AccessToken]').val(res.access_token);
$('input[name=accessToken]').val(res.access_token);
FB.api('me/feed', { access_token: res.access_token }, (result) => {
$('input[name=content]').val(JSON.stringify(result));
});
......@@ -31,6 +31,14 @@ Template.apiIndexationTpl.events({
alert('L\'ID de la page n\'est pas indiqué');
}
},
'click input[name=type]': (event) => {
const value = $(event.target).val();
if (value === 'facebook') {
$('.hidden-facebook').removeClass('hidden');
} else {
$('.hidden-facebook').addClass('hidden');
}
},
});
Template.apiIndexationTpl.helpers({
......
......@@ -37,12 +37,13 @@ Template.siteIndexationTpl.events({
$(event.target).find('h4').removeClass('hidden');
Meteor.callPromise('indexWebsite', url)
.then(() => {
.then((result) => {
console.log(result);
$(event.target).find('h4').addClass('hidden');
displayNotif({
type: 'success',
title: 'Indexation : ',
message: 'Indexation fini avec succès!',
message: `Indexation de ${url} fini avec succès avec ${result.items.length} pages indexées!`,
save: true,
});
}).catch((error) => {
......@@ -55,7 +56,6 @@ Template.siteIndexationTpl.events({
save: true,
});
});
event.target.reset();
},
'submit #formReglageSite': (event) => {
event.preventDefault();
......
......@@ -22,7 +22,8 @@
"simpl-schema": "^0.3.1",
"sitemapper": "^2.1.13",
"slug": "^0.9.1",
"sweetalert2": "^6.6.5"
"sweetalert2": "^6.6.5",
"twitter": "^1.7.1"
},
"devDependencies": {
"@meteorjs/eslint-config-meteor": "^1.0.5",
......
......@@ -784,6 +784,10 @@ deep-extend@^0.4.1, deep-extend@~0.4.0:
version "0.4.1"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253"
deep-extend@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.0.tgz#6ef4a09b05f98b0e358d6d93d4ca3caec6672803"
deep-is@~0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
......@@ -2393,7 +2397,7 @@ request-promise@^4.1.0:
stealthy-require "^1.1.0"
tough-cookie ">=2.3.0"
request@^2.74.0, request@^2.79.0, request@~2.79.0:
request@^2.72.0, request@^2.74.0, request@^2.79.0, request@~2.79.0:
version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies:
......@@ -2762,6 +2766,13 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0:
version "0.14.5"
resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
twitter@^1.7.1:
version "1.7.1"
resolved "https://registry.yarnpkg.com/twitter/-/twitter-1.7.1.tgz#0762378f1dc1c050e48f666aca904e24b1a962f4"
dependencies:
deep-extend "^0.5.0"
request "^2.72.0"
type-check@~0.3.2:
version "0.3.2"
resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment