Commit 1912705a authored by Nacim Goura's avatar Nacim Goura

Initial commit

parents
{
"extends": "eslint:recommended",
"plugins": [
"standard",
"promise"
],
"env": {
"browser": true,
"node": true,
"es6": true
},
"rules": {
"indent": ["error", 4, { "SwitchCase": 1 }],
"linebreak-style": ["error", "unix"],
"semi": ["error", "always"],
"no-console": "off"
}
}
\ No newline at end of file
/node_modules
.idea
\ No newline at end of file
idsearch
"use strict";
const express = require('express');
const path = require('path');
const logger = require('morgan');
const cookieParser = require('cookie-parser');
const bodyParser = require('body-parser');
const swig = require('swig');
let app = express();
let urlPublic = path.join(__dirname, 'public');
app.engine('html', swig.renderFile);
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'html');
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(require('node-sass-middleware')({
src: urlPublic,
dest: urlPublic,
outputStyle: 'compressed',
sourceMap: true
}));
app.use(express.static(urlPublic));
require('./routes/index')(app);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
let err = new Error('Not Found');
err.status = 404;
next(err);
});
// error handler
app.use(function(err, req, res) {
// set locals, only providing error in development
res.locals.message = err.message;
res.locals.error = req.app.get('env') === 'development' ? err : {};
// render the error page
res.status(err.status || 500);
res.render('error');
});
module.exports = app;
#!/usr/bin/env node
/**
* Module dependencies.
*/
const app = require('../app');
const debug = require('debug')('idsearch:server');
const http = require('http');
/**
* Get port from environment and store in Express.
*/
let port = normalizePort(process.env.PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
let server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? 'Pipe ' + port
: 'Port ' + port;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
let addr = server.address();
let bind = typeof addr === 'string'
? 'pipe ' + addr
: 'port ' + addr.port;
debug('Listening on ' + bind);
}
"use strict";
module.exports = {
addApi() {
}
};
"use strict";
const Sitemapper = require('sitemapper');
const Crawler = require("crawler");
const chalk = require('chalk');
const Q = require('q');
module.exports = {
getSiteMap(urlSiteMap) {
let sitemap = new Sitemapper();
let deferred = Q.defer();
sitemap.fetch(urlSiteMap).then((data) => {
deferred.resolve(data.sites);
}, function(err) {
deferred.reject(err);
});
return deferred.promise;
},
crawlUrl(listSite, term) {
let c = new Crawler({
maxConnections : 100,
// This will be called for each crawled page
callback : function (error, res, done) {
if(error){
console.log(error);
}else{
let $ = res.$;
let urlVisited = res.options.uri;
let titre = $("title").text().toLowerCase();
console.log("visited url : "+chalk.yellow(urlVisited));
console.log("titre : "+chalk.blue(titre));
getInternalLinks($);
let isWordFound = searchForWord($, term);
if(isWordFound) {
console.log('Word ' + chalk.green(term) + ' found ' + countOccurenceWord($, term) + ' time on this page ');
}
}
done();
}
});
c.queue(listSite);
}
};
function searchForWord($, word) {
let bodyText = $('html > body').text().toLowerCase();
return bodyText.indexOf(word.toLowerCase()) !== -1;
}
function countOccurenceWord($, word) {
let bodyText = $('html > body').text().toLowerCase();
let regex = new RegExp(word.toLowerCase(), 'g');
return (bodyText.match(regex) || []).length;
}
function getInternalLinks($) {
let allRelativeLinks = [];
let allAbsoluteLinks = [];
let relativeLinks = $("a[href^='/']");
relativeLinks.each(function() {
allRelativeLinks.push($(this).attr('href'));
});
let absoluteLinks = $("a[href^='http']");
absoluteLinks.each(function() {
allAbsoluteLinks.push($(this).attr('href'));
});
console.log("Found " + allRelativeLinks.length + " relative links");
console.log("Found " + allAbsoluteLinks.length + " absolute links");
}
\ No newline at end of file
{
"name": "idsearch",
"version": "0.0.0",
"private": true,
"scripts": {
"start": "DEBUG=idsearch* node ./bin/www"
},
"dependencies": {
"body-parser": "~1.16.0",
"chalk": "^1.1.3",
"cookie-parser": "~1.4.3",
"crawler": "^1.0.3",
"debug": "~2.6.0",
"express": "~4.14.1",
"morgan": "~1.7.0",
"node-sass-middleware": "^0.11.0",
"q": "^1.4.1",
"sitemapper": "^2.1.7",
"swig": "^1.4.2"
},
"devDependencies": {
"eslint": "^3.16.1",
"eslint-config-standard": "^7.0.0",
"eslint-plugin-promise": "^3.5.0",
"eslint-plugin-standard": "^2.1.0"
}
}
body{padding:50px;font:14px "Lucida Grande", Helvetica, Arial, sans-serif}a{color:#00B7FF}
/*# sourceMappingURL=style.css.map */
\ No newline at end of file
{
"version": 3,
"file": "style.css",
"sources": [
"style.scss"
],
"names": [],
"mappings": "AAAA,AAAA,IAAI,AAAC,CACH,OAAO,CAAE,IAAI,CACb,IAAI,CAAE,kDAAkD,CACzD,AAED,AAAA,CAAC,AAAC,CACA,KAAK,CAAE,OAAO,CACf"
}
\ No newline at end of file
body {
padding: 50px;
font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
}
a {
color: #00B7FF;
}
"use strict";
const crawler = require('../controller/crawlerController');
module.exports = function(app) {
app.route('/').get(function(req, res) {
crawler.getSiteMap('http://www.hautesavoie.fr/sitemap.xml')
.then(function(data) {
console.log(data);
}, function(err) {
console.log(err);
});
res.render('index', { title: 'Express' });
});
app.route('/').post(function(req, res) {
});
};
<h1>{{ message }}</h1>
<h2>{{ error.status }}</h2>
<pre>{{ error.stack }}</pre>
<!DOCTYPE html>
<html>
<head>
<title>{{ title }}</title>
<link rel='stylesheet' href='/stylesheets/style.css' />
</head>
<body>
<h1>{{ title }}</h1>
<p>Welcome to {{ title }}</p>
</body>
</html>
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment