Commit 46217035 authored by Nacim Goura's avatar Nacim Goura

crawlf and index facebook

parent 79120e83
......@@ -19,7 +19,7 @@ caching-compiler@1.1.9
caching-html-compiler@1.1.2
callback-hook@1.0.10
check@1.2.5
coffeescript@1.12.3_1
coffeescript@1.12.6_1
cosmos:browserify@0.10.0
dburles:collection-helpers@1.1.0
ddp@1.2.5
......@@ -47,7 +47,7 @@ html-tools@1.0.11
htmljs@1.0.11
http@1.2.12
id-map@1.0.9
johanbrook:publication-collector@1.0.7
johanbrook:publication-collector@1.0.8
jquery@1.11.10
kadira:blaze-layout@2.3.0
kadira:flow-router@2.12.1
......
import CrawFacebook from './crawlFacebook';
import CrawlFacebook from './crawlFacebook';
export default class crawlWebsite {
constructor(data) {
switch (data.type) {
case 'facebook':
return new CrawFacebook(data.tokenPage);
// facebook;
return new CrawlFacebook(data);
case 'twitter':
break;
......
import { Meteor } from 'meteor/meteor';
import FB from 'fb';
import _ from 'lodash';
import checkData from '../../utils/checkData';
export default class CrawlFacebook {
constructor(token) {
constructor(data) {
console.log('crawl Facebook');
this.listDataForIndex = [];
console.log('crawl facebook! ');
FB.setAccessToken(token);
return this.start();
if (data.content) {
this.content = JSON.parse(data.content);
return this.start();
}
throw new Meteor.Error('Error', 'aucune donnée Facebook!');
}
/**
*
* @returns Promise
*/
start() {
return new Promise((resolve, reject) => {
FB.api('me/feed', (res) => {
if (!res || res.error) {
reject(new Meteor.Error('Error', 'Token invalide!'));
}
if (res.data && res.data.length) {
this.parseData(res.data)
.then((listDataForIndex) => {
resolve(listDataForIndex);
});
} else {
reject(new Meteor.Error('Error', 'Aucune publication Facebook!'));
}
});
});
return this.parseData();
}
parseData(data) {
/**
*
* @returns Promise
*/
parseData() {
return new Promise((resolve) => {
console.log(data);
_.forEach(data, (item, index) => {
_.forEach(this.content.data, (item, index) => {
const dataForIndex = {
tag: 'social',
url: `https://www.facebook.com/${item.id}`,
description: '',
body: '',
};
......@@ -50,12 +46,12 @@ export default class CrawlFacebook {
dataForIndex.title_suggest = checkData.cleanText(item.name);
}
if (item.message) {
dataForIndex.description += checkData.cleanText(item.message) + ' ';
dataForIndex.body += checkData.cleanText(item.message) + ' ';
dataForIndex.description += `${checkData.cleanText(item.message)} `;
dataForIndex.body += `${checkData.cleanText(item.message)} `;
}
if (item.description) {
dataForIndex.description += checkData.cleanText(item.description) + ' ';
dataForIndex.body += checkData.cleanText(item.description) + ' ';
dataForIndex.description += `${checkData.cleanText(item.description)} `;
dataForIndex.body += `${checkData.cleanText(item.description)} `;
}
if (item.story) {
dataForIndex.description += checkData.cleanText(item.story);
......@@ -65,6 +61,10 @@ export default class CrawlFacebook {
dataForIndex.createdAt = item.created_time;
}
if (!dataForIndex.title && dataForIndex.description) {
dataForIndex.title = dataForIndex.description;
}
this.listDataForIndex.push({
index: {
_index: Meteor.settings.private.elasticsearch.esIndex,
......@@ -75,7 +75,7 @@ export default class CrawlFacebook {
this.listDataForIndex.push(dataForIndex);
if (index + 1 === data.length) {
if (index + 1 === this.content.data.length) {
resolve(this.listDataForIndex);
}
});
......
......@@ -4,9 +4,16 @@ import SimpleSchema from 'simpl-schema';
SimpleSchema.extendOptions(['autoform']);
export default new SimpleSchema({
tokenPage: {
AccessToken: {
type: String,
label: 'Token de la page',
autoform: {
type: 'hidden',
label: false,
},
},
idPage: {
type: String,
label: 'Id de la page',
},
type: {
type: String,
......@@ -28,4 +35,12 @@ export default new SimpleSchema({
},
},
},
content: {
type: String,
required: false,
autoform: {
type: 'hidden',
label: false,
},
},
}, { tracker: Tracker });
......@@ -6,6 +6,12 @@
<div class="panel-body">
<h4 class="text-center">Gestion de l'indexation des API</h4>
<hr>
<p>Pour pouvoir utiliser les API, il faut d'abord se connecter afin d'avoir les autorisations nécessaires!</p>
<p class="text-danger">L'id de la page doit être préalablement indiqué!</p>
<button id="loginButtonFacebook" class="btn btn-success">Login facebook</button>
<hr>
{{> quickForm id="formApiCrawl" schema=formApiCrawlSchema buttonContent="Valider" type="method" meteormethod="indexApi" }}
</div>
......
import { formApiCrawlSchema } from '/imports/collections/schemaForm';
import { Template } from 'meteor/templating';
import $ from 'jquery';
import './api.html';
Template.apiIndexationTpl.onRendered(() => {
$.ajaxSetup({ cache: true });
$.getScript('//connect.facebook.net/en_US/sdk.js', () => {
FB.init({
appId: '126994924549842',
version: 'v2.9',
});
});
});
Template.apiIndexationTpl.events({
'click #loginButtonFacebook': () => {
const inputIdPage = $('input[name=idPage]').val();
if (inputIdPage.length > 0) {
FB.login(() => {
FB.api(`${$('input[name=idPage]').val()}?fields=access_token`, (res) => {
$('input[name=AccessToken]').val(res.access_token);
FB.api('me/feed', { access_token: res.access_token }, (result) => {
$('input[name=content]').val(JSON.stringify(result));
});
});
}, { scope: 'manage_pages' });
} else {
alert('L\'ID de la page n\'est pas indiqué');
}
},
});
Template.apiIndexationTpl.helpers({
formApiCrawlSchema() {
return formApiCrawlSchema;
......
......@@ -36,7 +36,7 @@ ajv-keywords@^1.0.0:
version "1.5.1"
resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-1.5.1.tgz#314dd0a4b3368fad3dfcdc54ede6171b886daf3c"
ajv@^4.7.0, ajv@^4.9.1:
ajv@^4.7.0:
version "4.11.8"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536"
dependencies:
......@@ -67,10 +67,6 @@ ansi-styles@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
any-promise@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
aproba@^1.0.3:
version "1.1.2"
resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.1.2.tgz#45c6629094de4e96f693ef7eab74ae079c240fc1"
......@@ -506,10 +502,6 @@ caseless@~0.11.0:
version "0.11.0"
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.11.0.tgz#715b96ea9841593cc33067923f5ec60ebda4f7d7"
caseless@~0.12.0:
version "0.12.0"
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
center-align@^0.1.1:
version "0.1.3"
resolved "https://registry.yarnpkg.com/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad"
......@@ -651,10 +643,6 @@ copy-dereference@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/copy-dereference/-/copy-dereference-1.0.0.tgz#6b131865420fd81b413ba994b44d3655311152b6"
core-decorators@^0.17.0:
version "0.17.0"
resolved "https://registry.yarnpkg.com/core-decorators/-/core-decorators-0.17.0.tgz#3f43180a86d2ab0cc51069f46a1ec3e49e7cebd6"
core-js@^2.4.0:
version "2.4.1"
resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e"
......@@ -782,7 +770,7 @@ debug@2.6.7:
dependencies:
ms "2.0.0"
debug@^2.1.1, debug@^2.2.0, debug@^2.6.3:
debug@^2.1.1, debug@^2.2.0:
version "2.6.8"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc"
dependencies:
......@@ -792,7 +780,7 @@ decamelize@^1.0.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
deep-extend@0.4.1, deep-extend@^0.4.1, deep-extend@~0.4.0:
deep-extend@^0.4.1, deep-extend@~0.4.0:
version "0.4.1"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253"
......@@ -1219,7 +1207,7 @@ exit-hook@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8"
extend@~3.0.0:
extend@3.0.1, extend@~3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444"
......@@ -1231,16 +1219,6 @@ fast-levenshtein@~2.0.4:
version "2.0.6"
resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
fb@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/fb/-/fb-2.0.0.tgz#91fd40325da34ec41c68b25530fc3a3e0dacfa6a"
dependencies:
any-promise "^1.3.0"
babel-runtime "^6.23.0"
core-decorators "^0.17.0"
debug "^2.6.3"
request "^2.81.0"
figures@^1.3.5:
version "1.7.0"
resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
......@@ -1432,10 +1410,6 @@ handlebars@^4.0.4, handlebars@^4.0.5:
optionalDependencies:
uglify-js "^2.6"
har-schema@^1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e"
har-validator@~2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d"
......@@ -1445,13 +1419,6 @@ har-validator@~2.0.6:
is-my-json-valid "^2.12.4"
pinkie-promise "^2.0.0"
har-validator@~4.2.1:
version "4.2.1"
resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-4.2.1.tgz#33481d0f1bbff600dd203d75812a6a5fba002e2a"
dependencies:
ajv "^4.9.1"
har-schema "^1.0.5"
has-ansi@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91"
......@@ -2234,10 +2201,6 @@ pbkdf2@^3.0.3:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
performance-now@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5"
pify@^2.0.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
......@@ -2310,10 +2273,6 @@ qs@~6.3.0:
version "6.3.2"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.2.tgz#e75bd5f6e268122a2a0e0bda630b2550c166502c"
qs@~6.4.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
querystring-es3@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
......@@ -2459,33 +2418,6 @@ request@^2.74.0, request@^2.79.0, request@~2.79.0:
tunnel-agent "~0.4.1"
uuid "^3.0.0"
request@^2.81.0:
version "2.81.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
dependencies:
aws-sign2 "~0.6.0"
aws4 "^1.2.1"
caseless "~0.12.0"
combined-stream "~1.0.5"
extend "~3.0.0"
forever-agent "~0.6.1"
form-data "~2.1.1"
har-validator "~4.2.1"
hawk "~3.1.3"
http-signature "~1.1.0"
is-typedarray "~1.0.0"
isstream "~0.1.2"
json-stringify-safe "~5.0.1"
mime-types "~2.1.7"
oauth-sign "~0.8.1"
performance-now "^0.2.0"
qs "~6.4.0"
safe-buffer "^5.0.1"
stringstream "~0.0.4"
tough-cookie "~2.3.0"
tunnel-agent "^0.6.0"
uuid "^3.0.0"
require-uncached@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/require-uncached/-/require-uncached-1.0.3.tgz#4e0d56d6c9662fd31e43011c4b95aa49955421d3"
......@@ -2598,12 +2530,12 @@ signal-exit@^3.0.0:
version "3.0.2"
resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"
simpl-schema@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/simpl-schema/-/simpl-schema-0.3.0.tgz#8156ee5915860501afd1c30ca5c8278fb1152bfd"
simpl-schema@^0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/simpl-schema/-/simpl-schema-0.3.1.tgz#e55c3f3bc56e4edeee4e473843a08f4edb2ddbe1"
dependencies:
clone "1.0.2"
deep-extend "0.4.1"
extend "3.0.1"
message-box "0.0.2"
mongo-object "0.0.1"
underscore "1.8.3"
......@@ -2822,12 +2754,6 @@ tty-browserify@0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6"
tunnel-agent@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
dependencies:
safe-buffer "^5.0.1"
tunnel-agent@~0.4.1:
version "0.4.3"
resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment