Commit 54f75445 authored by Nacim Goura's avatar Nacim Goura

add skeleton

parent 1912705a
......@@ -13,6 +13,8 @@
"indent": ["error", 4, { "SwitchCase": 1 }],
"linebreak-style": ["error", "unix"],
"semi": ["error", "always"],
"no-console": "off"
"no-console": "off",
"curly": "error",
"eqeqeq": "error"
}
}
\ No newline at end of file
/node_modules
.idea
\ No newline at end of file
.idea
yarn.lock
/public/bower_components
\ No newline at end of file
......@@ -5,12 +5,15 @@ const path = require('path');
const logger = require('morgan');
const cookieParser = require('cookie-parser');
const bodyParser = require('body-parser');
const swig = require('swig');
const exphbs = require('express-handlebars');
let app = express();
let urlPublic = path.join(__dirname, 'public');
app.engine('html', swig.renderFile);
app.engine('html', exphbs({
defaultLayout : 'main',
extname : '.html'
}));
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'html');
......
......@@ -64,16 +64,16 @@ function onError(error) {
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
case 'EACCES':
console.error(bind + ' requires elevated privileges');
process.exit(1);
break;
case 'EADDRINUSE':
console.error(bind + ' is already in use');
process.exit(1);
break;
default:
throw error;
}
}
......
......@@ -3,6 +3,7 @@
const Sitemapper = require('sitemapper');
const Crawler = require("crawler");
const chalk = require('chalk');
const stripTags = require('strip-tags');
const Q = require('q');
module.exports = {
......@@ -20,25 +21,28 @@ module.exports = {
crawlUrl(listSite, term) {
let c = new Crawler({
maxConnections : 100,
maxConnections : 1000,
// This will be called for each crawled page
callback : function (error, res, done) {
if(error){
console.log(error);
}else{
}else if(res.statusCode === 200) {
let $ = res.$;
let body = removeTags($);
let urlVisited = res.options.uri;
let titre = $("title").text().toLowerCase();
console.log("visited url : "+chalk.yellow(urlVisited));
console.log("titre : "+chalk.blue(titre));
console.log($('h1').text());
getInternalLinks($);
let isWordFound = searchForWord($, term);
let isWordFound = searchForWord(body, term);
if(isWordFound) {
console.log('Word ' + chalk.green(term) + ' found ' + countOccurenceWord($, term) + ' time on this page ');
console.log('Word ' + chalk.green(term) + ' found ' + countOccurenceWord(body, term) + ' time on this page ');
}
}
done();
......@@ -49,17 +53,39 @@ module.exports = {
};
function searchForWord($, word) {
/**
* remove useless tags
*/
function removeTags($) {
let bodyText = $('html > body').text().toLowerCase();
return bodyText.indexOf(word.toLowerCase()) !== -1;
return stripTags(bodyText, ['link', 'script']);
}
function countOccurenceWord($, word) {
let bodyText = $('html > body').text().toLowerCase();
/**
* search term in body
* @param data
* @param word
* @returns {boolean}
*/
function searchForWord(data, word) {
return data.indexOf(word.toLowerCase()) !== -1;
}
/**
* count occurence terms
* @param data
* @param word
* @returns {Number}
*/
function countOccurenceWord(data, word) {
let regex = new RegExp(word.toLowerCase(), 'g');
return (bodyText.match(regex) || []).length;
return (data.match(regex) || []).length;
}
/**
* search internal links in analyzed page
* @param $ dom jquery
*/
function getInternalLinks($) {
let allRelativeLinks = [];
let allAbsoluteLinks = [];
......
"use strict";
const elasticsearch = require('elasticsearch');
const esClient = new elasticsearch.Client({
host: '127.0.0.1:9200',
log: 'error'
});
\ No newline at end of file
......@@ -3,7 +3,8 @@
"version": "0.0.0",
"private": true,
"scripts": {
"start": "DEBUG=idsearch* node ./bin/www"
"start": "DEBUG=idsearch* node ./bin/www",
"postinstall": "cd public && bower install"
},
"dependencies": {
"body-parser": "~1.16.0",
......@@ -11,12 +12,14 @@
"cookie-parser": "~1.4.3",
"crawler": "^1.0.3",
"debug": "~2.6.0",
"elasticsearch": "^12.1.3",
"express": "~4.14.1",
"express-handlebars": "^3.0.0",
"morgan": "~1.7.0",
"node-sass-middleware": "^0.11.0",
"q": "^1.4.1",
"sitemapper": "^2.1.7",
"swig": "^1.4.2"
"strip-tags": "^0.1.1"
},
"devDependencies": {
"eslint": "^3.16.1",
......
{
"name": "public",
"version": "0.0.1",
"dependencies": {
"bootstrap-sass": "^3.3.7"
}
}
This diff is collapsed.
@import "./_variables.scss";
@import "../bower_components/bootstrap-sass/assets/stylesheets/bootstrap";
@font-face {
font-family: 'Glyphicons Halflings';
src: url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.eot");
src: url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.eot?") format("embedded-opentype"),
url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.woff2") format("woff2"),
url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.woff") format("woff"),
url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.ttf") format("truetype"),
url("../bower_components/bootstrap-sass/assets/fonts/bootstrap/glyphicons-halflings-regular.svg") format("svg");
}
/* CSS declarations go here */
body {
margin-top: 100px;
overflow: hidden;
}
form {
margin-bottom: 20px;
}
@media (min-width: $grid-float-breakpoint) {
.container-fluid {
margin-left: 42px;
}
}
.main {
padding-top: 5px;
}
.wrapper {
position: relative;
overflow: auto;
background-color: #ffffff;
padding-top: 15px;
}
.border-left {
border-left: 1px solid $table-border-color;
}
.navbar-inverse .navbar-brand {
font-size: 24px;
}
.two-cols {
column-count: 2;
}
.fa {
font-size: 16px;
}
.two-columns {
column-count: 2;
}
.view-group-addon .fa,
.delete-group-addon .fa {
color: #ffffff;
}
.radio-select-inline > div,
.radio-select-inline .radio {
display: inline-block;
vertical-align: bottom;
}
.radio-select-inline label:first-child {
padding-right: 10px;
}
.rc-percentage {
margin-bottom: 0;
width: 120px
}
/* TODO: fix problem */
.dataTables_processing {
display: none !important;
}
.panel-footer {
background: transparent;
}
.input-group-addon {
color: #fff;
}
.max-height {
height: 100%;
overflow: auto;
}
.max-width {
width: 100%;
}
.half-width {
width: 50%;
}
.clear {
overflow: auto;
}
.btn-interface {
padding: 5px 2px 3px 2px;
position: absolute;
top: 0;
}
.btn-interface-caption {
position: relative;
top: 0;
margin-right: 5px;
}
.caption-title {
font-weight: bold;
color: $gray-dark;
}
.label-display {
font-weight: bold;
}
.nav-tabs {
border-bottom-width: 0;
}
.radio,
.checkbox {
margin: 0;
}
table.dataTable.document-table {
margin-top: 0 !important;
caption {
padding-top: 0;
}
}
.dataTable tr {
cursor: pointer !important;
}
.dataTable tbody tr:hover {
background-color: lighten($brand-primary, 35%) !important;
}
.relative {
position: relative;
}
span.link {
color: $brand-primary;
text-decoration: underline;
cursor: pointer;
}
.modal-header {
border-radius: 6px 6px 0 0;
}
.big-modal {
width: 75% !important;
height: 80vh !important;
margin: 10vh auto !important;
.modal-body {
position: absolute;
width: 100%;
top: 50px;
bottom: 0;
overflow-y: auto;
}
.tab-content {
position: absolute;
top: 65px;
bottom: 0;
overflow-y: auto;
}
.tab-pane {
padding-bottom: 60px;
}
.btn-region {
position: absolute;
bottom: 15px;
right: 15px;
left: 15px;
}
}
.movable {
cursor: move;
}
.not-movable {
cursor: no-drop;
}
.warning {
@extend .bg-warning;
}
.danger {
@extend .bg-danger;
}
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
body {
padding: 50px;
font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
}
a {
color: #00B7FF;
}
@import "./main.scss";
\ No newline at end of file
......@@ -6,15 +6,20 @@ module.exports = function(app) {
app.route('/').get(function(req, res) {
crawler.getSiteMap('http://www.hautesavoie.fr/sitemap.xml')
.then(function(data) {
console.log(data);
.then(function(sites) {
crawler.crawlUrl(sites, 'test');
}, function(err) {
console.log(err);
});
res.render('index', { title: 'Express' });
});
app.route('/').post(function(req, res) {
app.route('/api').post(function(req, res) {
crawler.getSiteMap('http://www.hautesavoie.fr/sitemap.xml')
.then(function(data) {
console.log(data);
}, function(err) {
console.log(err);
});
});
};
<!DOCTYPE html>
<html>
<head>
<title>{{ title }}</title>
<link rel='stylesheet' href='/stylesheets/style.css' />
</head>
<body>
<h1>{{ title }}</h1>
<p>Welcome to {{ title }}</p>
</body>
</html>
<div class="row">
<form action="/api" method="post">
<input type="search" class="form-control" placeholder="Rechercher...">
<input type="submit" class="btn btn-success" value="Rechercher">
</form>
</div>
<!DOCTYPE html>
<html>
<head>
<title>{{ title }}</title>
<link rel='stylesheet' href='/stylesheets/style.css' />
</head>
<body>
{{> header}}
<div class="container">
{{{body}}}
</div>
<script src="/bower_components/jquery/dist/jquery.slim.js"></script>
<script src="/bower_components/bootstrap-sass/assets/javascripts/bootstrap.min.js"></script>
</body>
</html>
<nav role="navigation" class="navbar navbar-inverse navbar-fixed-top">
<div class="container-fluid">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar-ex1-collapse" aria-expanded="false">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="/">IdSearch</a>
</div>
<ul class="nav navbar-nav">
<li>
<a class="" href="/admin" ><i class="fa fa-lock"></i> Admin</a>
</li>
</ul>
</div>
</nav>
\ No newline at end of file
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment