Skip to content

Commit

Permalink
refactoring: a same attribute name can't be in different types of a s…
Browse files Browse the repository at this point in the history 
…ame index for elasticsearch... Therefore we need to create a specific index for each template.
  • Loading branch information
@gregorybesson
gregorybesson committed May 1, 2017
1 parent f917ed4 commit 2388197
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 78 deletions.
29 changes: 18 additions & 11 deletions hooks/hooks.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
'use strict'
var path = require('path');
var esconnection = require('../modules/esconnection');
var path = require('path')
var esconnection = require('../modules/esconnection')

var hooks = {
afterPublish: function (result, postPath, abe) {
Expand All @@ -13,16 +13,23 @@ var hooks = {

if(abe.config.elasticsearch.templates){
if(abe.config.elasticsearch.templates.indexOf(template) > -1) {
es.client.index({
index: es.index,
id: link,
type: template,
body: content
this.client.indices.create({
index: this.index + '_' + template
},function(err,resp,status) {
if(err && err.statusCode !== 400) {
console.log(err);
}
es.client.index({
index: es.index + '_' + template,
id: link,
type: template,
body: content
});
});
}
} else {
es.client.index({
index: es.index,
index: es.index + '_' + template,
id: link,
type: template,
body: content
Expand All @@ -38,8 +45,8 @@ var hooks = {
const link = json.abe_meta.link
const template = json.abe_meta.template

es.client.delete({
index: es.index,
var result = es.client.delete({
index: es.index + '_' + template,
id: link,
type: template
});
Expand All @@ -51,7 +58,7 @@ var hooks = {
if(abe.config.elasticsearch && abe.config.elasticsearch.active){
var es = new esconnection(abe)
es.client.delete({
index: es.index,
index: es.index + '_' + template,
id: path
});
}
Expand Down
78 changes: 71 additions & 7 deletions modules/esconnection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ function esclient (abe) {
var host = "localhost"
var port = 9200
this.index = path.basename(abe.config.root)
this.error = null
this._pathTemplate = path.join(abe.config.root, abe.config.templates.url)
this._extension = '.' + abe.config.files.templates.extension

if(abe.config.elasticsearch){
var elt = abe.config.elasticsearch
Expand All @@ -19,13 +22,74 @@ function esclient (abe) {
log: 'error'
})

this.client.indices.create({
index: this.index
},function(err,resp,status) {
if(err && err.statusCode !== 400) {
console.log(err);
}
});
this.indices = []
// this.error = this.client.ping({
// requestTimeout: 3000,
// hello: "elasticsearch!"
// }, function (error) {
// if (error) {
// console.trace('elasticsearch cluster is down!')
// return error
// }
// })

this.getIndices = function(templates, extension, callback){
Array.prototype.forEach.call(templates, function(template) {
template = path.basename(template, extension)
const index = this.index + '_' + template
this.client.indices.exists({ index: index, ignoreUnavailable: true }, function (err, exists) {
if(exists === true){
this.indices.push(index)
}
callback(this.indices)
}.bind(this))
}.bind(this))
}

this.initIndices = function(templates, extension, callback){
Array.prototype.forEach.call(templates, (template) => {
template = path.basename(template,extension)
if(this.isInIndices(template)){
this.client.indices.create({
index: this.index + '_' + template
},function(err,resp,status) {
callback(resp)
if(err && err.statusCode !== 400) {
console.log(err);
}
})
} else {
callback(true)
}
})
}

this.resetIndices = function(templates, extension, callback){
Array.prototype.forEach.call(templates, (template) => {
template = path.basename(template,extension)
const index = this.index + '_' + template
this.client.indices.exists({ index : index, ignoreUnavailable: true }, function (err, exists) {
if(exists === true){
this.client.indices.delete({index: index});
}
callback(index)
}.bind(this))
})
}

this.isInIndices = function(template){
if(abe.config.elasticsearch && abe.config.elasticsearch.active){
if(abe.config.elasticsearch.templates){
if(abe.config.elasticsearch.templates.indexOf(template) > -1) {
return true
}
} else {
return true
}
}

return false
}
}

module.exports = esclient;
74 changes: 45 additions & 29 deletions partials/console.html
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,50 @@
<!DOCTYPE html>
<html>
<html data-config='{"URL":"{{abeUrl}}","EXTENSION":"{{@root.config.files.templates.extension}}","TPLPATH":"{{@root.json.abe_meta.template}}","TPLNAME":"{{tplName}}","FILEPATH":"{{@root.json.abe_meta.link}}"}'
data-slugs='{{{printJson @root.slugs}}}'
data-json='{{{printJson json 1}}}'
data-Locales='{{{printJson Locales}}}' >
<head>
<title></title>
<link rel="stylesheet" href="/css/bootstrap.min.css">
<link rel="stylesheet" href="/css/styles.css">
<script type="text/javascript" src="/libs/jquery-2.1.0.min.js"></script>
<script type="text/javascript" src="/libs/bootstrap.min.js"></script>
</head>
<body>

<div class="row col-xs-offset-1 col-xs-10">
<ol class="breadcrumb">
<li><a href="/abe/">Home</a></li>
<li><a href="/abe/plugin/abe-elasticsearch/bulk" class="active">Elasticsearch admin</a></li>
</ol>
<title>Abe</title>
{{abeImport 'common-styles' manager.config this}}
</head>
<body class="with-menu engine-open">
{{abeImport 'header-menu' manager.config this}}
<div class="container-main">
<div class="container-table">
<div>
<div class="no-gutter manager-wrapper visible">
<div class="manager-left form-wrapper">
{{abeImport 'main-menu' manager.config this}}
</div>

<div class="alert alert-danger hidden" role="alert">

</div>

<div class="form-group">
<button onclick="location.href='?reindex=true'" class="btn btn-default">Index your blog with Elasticsearch</button>
</div>
{{#if nbIndexed}}
<br/>
<div>{{nbIndexed}} documents have been indexed in Elasticsearch</div>
<br/>
{{/if}}
<div class="manager-right tab">

<br /><br />
</div>
</body>
<div class="manager-tab structure-folders">
<div class="text-uppercase">
Elastic Search Console
</div>
<div class="manager-tab-content">
<div class="form-group">
<button onclick="location.href='?reindex=true'" class="btn btn-default">Index your blog with Elasticsearch</button>
</div>
{{#if nbIndexed}}
<br/>
<div>{{nbIndexed}} documents have been indexed in Elasticsearch</div>
<br/>
{{/if}}
{{#if error}}
<h1> {{error}} </h1>
{{/if}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
{{abeImport 'common-scripts' manager.config this}}
<script type="text/javascript" src="/abecms/scripts/admin-compiled.js"></script>
<script type="text/javascript" src="/abecms/scripts/template-engine-compiled.js"></script>
</body>
</html>
98 changes: 67 additions & 31 deletions routes/get/console.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,43 +8,79 @@ var route = function route(req, res, next, abe) {

var nbIndexed
var es = new esconnection(abe)
var manager = {}
manager.home = {files: []}
manager.config = JSON.stringify(abe.config)

if(req.query.reindex === "true"){
es.client.indices.delete({index: es.index});
var files = abe.Manager.instance.getListWithStatusOnFolder('publish')
Array.prototype.forEach.call(files, (fileObj) => {
var isHome = true
var jsonPath = null
var linkPath = null
var template = null
var fileName = null
var folderPath = null
var EditorVariables = {
nbIndexed: 0,
user: res.user,
slugs: abe.Manager.instance.getSlugs(),
abeUrl: '/abe/editor/',
Locales: abe.coreUtils.locales.instance.i18n,
manager: manager,
config: abe.config
}
const pathTemplate = path.join(abe.config.root, abe.config.templates.url)
const extension = '.' + abe.config.files.templates.extension
let templates = abe.coreUtils.file.getFilesSync(pathTemplate, true, extension)

const revisionPath = path.join(abe.config.root, abe.config.data.url, fileObj.abe_meta.link.replace(`.${abe.config.files.templates.extension}`, '.json'))
const link = fileObj.abe_meta.link
const template = fileObj.abe_meta.template
const content = abe.cmsData.file.get(revisionPath)
// if(es.error !== null){
// //nbIndexed: resp.count,
// console.log(es.error)
// EditorVariables.error = "can't connect to the elasticsearch server"
// res.render(path.join(__dirname + '/../../partials/console.html'), EditorVariables)
// return
// }

es.client.index({
index: es.index,
id: link,
type: template,
body: content
})
if(req.query.reindex === "true"){
let resetWaiting = templates.length;
es.resetIndices(templates, extension, function(indices){
resetWaiting--;
if (resetWaiting==0) {
let resetInit = templates.length;
es.initIndices(templates, extension, function(resp){
resetInit--;
if (resetInit==0) {
const files = abe.Manager.instance.getListWithStatusOnFolder('publish')
Array.prototype.forEach.call(files, (fileObj) => {
const revisionPath = path.join(abe.config.root, abe.config.data.url, fileObj.abe_meta.link.replace(`.${abe.config.files.templates.extension}`, '.json'))
const link = fileObj.abe_meta.link
const template = fileObj.abe_meta.template
const content = abe.cmsData.file.get(revisionPath)
const index = es.index + '_' + template
if(es.isInIndices(template)){
es.client.index({
index: index,
id: link,
type: template,
body: content
})
}
})
}
})
}
})
}

es.client.count({index: es.index},function(err,resp,status) {
var htmlToSend = '';

var data = path.join(__dirname + '/../../partials/console.html')
var html = abe.coreUtils.file.getContent(data);

var template = abe.Handlebars.compile(html, {noEscape: true})
var tmp = template({
express: {
req: req,
res: res
},
nbIndexed: resp.count
})

return res.send(tmp);
});
let getWaiting = templates.length;
es.getIndices(templates, extension, function(indices){
getWaiting--;
if (getWaiting==0) {
es.client.count({index: indices.join()},function(err,resp,status) {
EditorVariables.nbIndexed = resp.count
res.render(path.join(__dirname + '/../../partials/console.html'), EditorVariables)
}.bind(this));
}
}.bind(this))

}

exports.default = route

0 comments on commit 2388197

Please sign in to comment.