diff --git a/package-lock.json b/package-lock.json index d775457..551a4d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "template-nodejs-elasticsearch-microservice", + "name": "x5gon-elasticsearch-microservice", "version": "1.0.0", "lockfileVersion": 1, "requires": true, @@ -138,15 +138,6 @@ "@types/range-parser": "*" } }, - "@types/express-validator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/express-validator/-/express-validator-3.0.0.tgz", - "integrity": "sha512-LusnB0YhTXpBT25PXyGPQlK7leE1e41Vezq1hHEUwjfkopM1Pkv2X2Ppxqh9c+w/HZ6Udzki8AJotKNjDTGdkQ==", - "dev": true, - "requires": { - "express-validator": "*" - } - }, "@types/express-winston": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@types/express-winston/-/express-winston-4.0.0.tgz", diff --git a/package.json b/package.json index 4548134..d116cd8 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,6 @@ "@types/cookie-parser": "^1.4.2", "@types/elasticsearch": "^5.0.36", "@types/express": "^4.17.3", - "@types/express-validator": "^3.0.0", "@types/express-winston": "^4.0.0", "@types/node": "^13.9.2", "mocha": "^7.1.1", diff --git a/src/Interfaces.ts b/src/Interfaces.ts index d0a9767..fbffb25 100644 --- a/src/Interfaces.ts +++ b/src/Interfaces.ts @@ -20,4 +20,70 @@ export interface IConfiguration { elasticsearch: { node: string } +} + +//////////////////////////////////////////////////////////////////////// +// Elasticsearch records + +export interface IContent { + content_id: number +} + +export interface IWikipedia { + uri: string, + name: string, + sec_uri: string, + sec_name: string +} + +export interface IElasticsearchHit { + _score: number, + _source: { + material_id: number, + title: string, + description: string, + creation_date: string, + retrieved_date: string, + type: string, + mimetype: string, + material_url: string, + website_url: string, + language: string, + license: { + short_name: string, + typed_name?: string[], + disclaimer: string, + url: string + }, + contents?: IContent[], + provider_id: number, + provider_name: string, + provider_url: string, + wikipedia: IWikipedia[] + } +} + + +//////////////////////////////////////////////////////////////////////// +// Express API routes + +export interface ISearch { + text?: string, + url?: string, + types?: string, + licenses?: string, + languages?: string, + content_languages?: string, + provider_ids?: string, + wikipedia?: boolean, + wikipedia_limit?: number, + limit?: number, + page?: number +} + +export interface IQueryElement { + term?: object, + terms?: object, + regexp?: object, + exists?: object } \ No newline at end of file diff --git a/src/load/create-elasticsearch-index.js b/src/load/create-elasticsearch-index.js index 23ef649..10722ec 100644 --- a/src/load/create-elasticsearch-index.js +++ b/src/load/create-elasticsearch-index.js @@ -28,7 +28,7 @@ function materialType(mimetype) { } // merges all of the material information into a single line -const pg_command = ` +const pgCommand = ` WITH URLS AS ( SELECT COALESCE(m.material_id, c.material_id) AS material_id, @@ -162,7 +162,7 @@ async function populate() { console.log("executing pg-command"); - pg.executeLarge(pg_command, [], 100, + pg.executeLarge(pgCommand, [], 100, (error, records, callback) => { if (error) { console.log(error); return; } @@ -179,26 +179,26 @@ async function populate() { // modify the license attribute when sending to elasticsearch const url = record.license; - let short_name; - let typed_name; + let shortName; + let typedName; let disclaimer = DEFAULT_DISCLAIMER; if (url) { const regex = /\/licen[sc]es\/([\w\-]+)\//; - short_name = url.match(regex)[1]; - typed_name = short_name.split("-"); + shortName = url.match(regex)[1]; + typedName = shortName.split("-"); } else { - short_name = NO_LICENSE_DISCLAIMER; + shortName = NO_LICENSE_DISCLAIMER; } record.license = { - short_name, - typed_name, + short_name: shortName, + typed_name: typedName, disclaimer, url }; // modify the wikipedia array - for (let value of record.wikipedia) { + for (const value of record.wikipedia) { // rename the wikipedia concepts value.sec_uri = value.secUri; value.sec_name = value.secName; @@ -221,7 +221,7 @@ async function populate() { count++; xcallback(null); }) - .catch((error) => xcallback); + .catch(xcallback); }); } diff --git a/src/routes/v1/index.ts b/src/routes/v1/index.ts index b2297fa..d91ab76 100644 --- a/src/routes/v1/index.ts +++ b/src/routes/v1/index.ts @@ -1,9 +1,12 @@ import { Express } from "express"; import { IConfiguration } from "../../Interfaces"; +import search from "./oer_materials"; +import recommend from "./recommend"; + // join all routers in a single function export default function index(app: Express, config: IConfiguration) { // setup the microservices API routes - app.use("/api/v1", require("./oer_materials")(config)); - app.use("/api/v1", require("./recommend")(config)); + app.use("/api/v1", search(config)); + app.use("/api/v1", recommend(config)); }; diff --git a/src/routes/v1/oer_materials.js b/src/routes/v1/oer_materials.ts similarity index 76% rename from src/routes/v1/oer_materials.js rename to src/routes/v1/oer_materials.ts index 1faafb7..a4bae9c 100644 --- a/src/routes/v1/oer_materials.js +++ b/src/routes/v1/oer_materials.ts @@ -4,26 +4,27 @@ * manipuating data from Elastic Search. */ -const router = require("express").Router(); -const { default: Elasticsearch } = require("../../library/elasticsearch"); -const { ErrorHandler } = require("../../library/error"); +import { IConfiguration, IElasticsearchHit, ISearch, IQueryElement } from "../../Interfaces"; -// internal modules -const mimetypes = require("../../config/mimetypes"); +import { Router, Request, Response, NextFunction } from "express"; // validating the query parameters -const { query, body, param } = require("express-validator"); - +import { query, param } from "express-validator"; // creation of the query string to help the user navigate through -const querystring = require("querystring"); - -/** - * Returns the general material type. - * @param {String} mimetype - The document mimetype. - * @returns {String|Null} The material type. - */ -function materialType(mimetype) { - for (let type in mimetypes) { +import * as querystring from "querystring"; +// add error handling functionality +import { ErrorHandler } from "../../library/error"; +// import elasticsearch module +import Elasticsearch from "../../library/elasticsearch"; +// import file mimetypes lists +import * as mimetypes from "../../config/mimetypes.json"; + +// initialize the express router +const router = Router(); + +// returns the general material type +function materialType(mimetype: string) { + for (const type in mimetypes) { if (mimetypes[type].includes(mimetype)) { return type; } @@ -31,7 +32,8 @@ function materialType(mimetype) { return null; } -function materialFormat(hit, { wikipedia, wikipedia_limit }) { +// format the material +function materialFormat(hit: IElasticsearchHit, wikipedia?: boolean, wikipedia_limit?: number) { return { weight: hit._score, material_id: hit._source.material_id, @@ -48,9 +50,9 @@ function materialFormat(hit, { wikipedia, wikipedia_limit }) { provider: { id: hit._source.provider_id, name: hit._source.provider_name.toLowerCase(), - domain: hit._source.provider_url, + domain: hit._source.provider_url }, - content_ids: hit._source.contents ? hit._source.contents.map((content) => content.content_id) : [], + content_ids: hit._source.contents ? hit._source.contents.map(content => content.content_id) : [], ...wikipedia && { wikipedia: wikipedia_limit && wikipedia_limit > 0 ? hit._source.wikipedia.slice(0, wikipedia_limit) @@ -59,12 +61,8 @@ function materialFormat(hit, { wikipedia, wikipedia_limit }) { }; } -/** - * @description Assign the elasticsearch API routes. - * @param {Object} config - The configuration object. - * @returns {Object} The search router. - */ -module.exports = (config) => { +// assign the elasticsearch API routes +export default (config: IConfiguration) => { // set the default parameters const DEFAULT_LIMIT = 20; const MAX_LIMIT = 100; @@ -85,40 +83,39 @@ module.exports = (config) => { router.get("/oer_materials", [ query("text").trim(), query("types").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase() : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase() : null)), query("licenses").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("languages").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("content_languages").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("provider_ids").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",").map((id) => parseInt(id, 10)) : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",").map(id => parseInt(id, 10)) : null)), query("wikipedia").optional().toBoolean(), query("wikipedia_limit").optional().toInt(), query("limit").optional().toInt(), - query("page").optional().toInt(), - ], async (req, res, next) => { + query("page").optional().toInt() + ], async (req: Request, res: Response, next: NextFunction) => { // extract the appropriate query parameters - let { - query: { - text, - types, - languages, - content_languages, - provider_ids, - licenses, - wikipedia, - wikipedia_limit, - limit, - page, - } - } = req; + const requestQuery: ISearch = req.query; + const { + text, + types, + languages, + content_languages, + provider_ids, + licenses, + wikipedia, + wikipedia_limit, + limit: queryLimit, + page: queryPage + } = requestQuery; if (!text) { return res.status(400).json({ message: "query parameter 'text' not available", - query: req.query + query: requestQuery }); } @@ -127,27 +124,34 @@ module.exports = (config) => { // ------------------------------------ // set default pagination values - if (!limit) { - limit = DEFAULT_LIMIT; - } else if (limit <= 0) { - limit = DEFAULT_LIMIT; - } else if (limit >= MAX_LIMIT) { - limit = MAX_LIMIT; - } + // which part of the materials do we want to query + const limit: number = !queryLimit + ? DEFAULT_LIMIT + : queryLimit <= 0 + ? DEFAULT_LIMIT + : queryLimit >= MAX_LIMIT + ? DEFAULT_LIMIT + : queryLimit; + + const page: number = !queryPage + ? DEFAULT_PAGE + : queryPage; + + const size = limit; + const from = (page - 1) * size; + req.query.limit = limit; - if (!page) { - page = DEFAULT_PAGE; - req.query.page = page; - } + req.query.page = page; // ------------------------------------ // Set query parameters // ------------------------------------ // set the nested must conditions for the "contents" attribute - const nestedContentsMust = [{ + const nestedContentsMust: IQueryElement[] = [{ term: { "contents.extension": "plain" } }]; + if (content_languages) { nestedContentsMust.push({ terms: { "contents.language": content_languages } @@ -159,16 +163,16 @@ module.exports = (config) => { // ------------------------------------ // get the filter parameters (type and language) - let typegroup; - let filetypes; + let typegroup: string; + let filetypes: string; if (types && ["all", "text", "video", "audio"].includes(types)) { typegroup = types === "all" ? null : types; } else if (types && types.split(",").length > 0) { - filetypes = types.split(",").map((t) => `.*\.${t.trim()}`).join("|"); + filetypes = types.split(",").map((t:string) => `.*\.${t.trim()}`).join("|"); } // add the filter conditions for the regex - const filters = []; + const filters: IQueryElement[] = []; if (filetypes) { filters.push({ regexp: { material_url: filetypes } @@ -205,17 +209,13 @@ module.exports = (config) => { } // check if we need to filter the documents - const filterFlag = filters.length; - - // which part of the materials do we want to query - const size = limit; - const from = (page - 1) * size; + const filterFlag = filters.length > 0; // ------------------------------------ // Translate the user input // ------------------------------------ - let translation = text; + const translation = text; // ------------------------------------ // Set the elasticsearch query body @@ -253,7 +253,7 @@ module.exports = (config) => { query: { bool: { must: [ - { match: { "wikipedia.sec_name": translation } }, + { match: { "wikipedia.sec_name": translation } } ] } } @@ -286,18 +286,20 @@ module.exports = (config) => { // get the search results from elasticsearch const results = await es.search("oer_materials", esQuery); // format the output before sending - const output = results.hits.hits.map((hit) => materialFormat(hit, { wikipedia, wikipedia_limit })); + const output = results.hits.hits.map((hit: IElasticsearchHit) => + materialFormat(hit, wikipedia, wikipedia_limit) + ); // prepare the parameters for the previous query const prevQuery = { ...req.query, - ...page && { page: page - 1 }, + ...page && { page: page - 1 } }; // prepare the parameters for the next query const nextQuery = { ...req.query, - ...page && { page: page + 1 }, + ...page && { page: page + 1 } }; const BASE_URL = "https://platform.x5gon.org/api/v1/search"; @@ -306,7 +308,7 @@ module.exports = (config) => { const totalPages = Math.ceil(results.hits.total.value / size); const prevPage = page - 1 > 0 ? `${BASE_URL}?${querystring.stringify(prevQuery)}` : null; const nextPage = totalPages >= page + 1 ? `${BASE_URL}?${querystring.stringify(nextQuery)}` : null; - results.aggregations.providers.buckets.forEach((provider) => { + results.aggregations.providers.buckets.forEach((provider: { key: string }) => { provider.key = provider.key.toLowerCase(); }); @@ -321,9 +323,9 @@ module.exports = (config) => { next_page: nextPage, aggregations: { licenses: results.aggregations.licenses.buckets, - types: results.aggregations.types.buckets, languages: results.aggregations.languages.buckets, - providers: results.aggregations.providers.buckets + providers: results.aggregations.providers.buckets, + types: results.aggregations.types.buckets } } }); @@ -338,7 +340,7 @@ module.exports = (config) => { * @apiName esSearchAPI * @apiGroup search */ - router.post("/oer_materials", async (req, res, next) => { + router.post("/oer_materials", async (req: Request, res: Response, next: NextFunction) => { const { body: { record } } = req; @@ -353,9 +355,9 @@ module.exports = (config) => { // modify the license attribute when sending to elasticsearch const url = record.license; - let shortName; - let typedName; - let disclaimer = DEFAULT_DISCLAIMER; + let shortName: string; + let typedName: string[]; + const disclaimer = DEFAULT_DISCLAIMER; if (url) { const regex = /\/licen[sc]es\/([\w\-]+)\//; @@ -372,7 +374,7 @@ module.exports = (config) => { }; // modify the wikipedia array - for (let value of record.wikipedia) { + for (const value of record.wikipedia) { // rename the wikipedia concepts value.sec_uri = value.secUri; value.sec_name = value.secName; @@ -394,20 +396,24 @@ module.exports = (config) => { // refresh the index after pushing the new record await es.refreshIndex("oer_materials"); // return the material id of the added record - return res.status(200).json({ message: "record pushed to the index" }); + return res.status(200).json({ + message: "record pushed to the index", + material_id: record.material_id + }); } catch (error) { return next(new ErrorHandler(500, "Internal server error")); } }); + // get particular material router.get("/oer_materials/:material_id", [ param("material_id").toInt(), query("wikipedia").optional().toBoolean(), - query("wikipedia_limit").optional().toInt(), - ], async (req, res, next) => { + query("wikipedia_limit").optional().toInt() + ], async (req: Request, res: Response, next: NextFunction) => { const { params: { - material_id + material_id: materialId }, query: { wikipedia, @@ -415,19 +421,22 @@ module.exports = (config) => { } } = req; - if (!material_id) { + if (!materialId) { return res.status(400).json({ message: "body parameter material_id not an integer", - query: { material_id } + params: req.params, + query: req.query }); } try { const results = await es.search("oer_materials", { - query: { terms: { _id: [material_id] } } + query: { terms: { _id: [materialId] } } }); // format the output before sending - const output = results.hits.hits.map((hit) => materialFormat(hit, { wikipedia, wikipedia_limit }))[0]; + const output = results.hits.hits.map((hit: IElasticsearchHit) => + materialFormat(hit, wikipedia, wikipedia_limit) + )[0]; // return the status as the response return res.status(200).json({ @@ -445,8 +454,8 @@ module.exports = (config) => { * @apiGroup search */ router.patch("/oer_materials/:material_id", [ - param("material_id").toInt(), - ], async (req, res, next) => { + param("material_id").toInt() + ], async (req: Request, res: Response, next: NextFunction) => { const { params: { material_id }, body: { record } @@ -461,7 +470,7 @@ module.exports = (config) => { try { // modify the wikipedia array - for (let value of record.wikipedia) { + for (const value of record.wikipedia) { // rename the wikipedia concepts value.sec_uri = value.secUri; value.sec_name = value.secName; @@ -497,7 +506,7 @@ module.exports = (config) => { */ router.delete("/oer_materials/:material_id", [ param("material_id").toInt() - ], async (req, res, next) => { + ], async (req: Request, res: Response, next: NextFunction) => { const { params: { material_id } } = req; diff --git a/src/routes/v1/recommend.js b/src/routes/v1/recommend.ts similarity index 63% rename from src/routes/v1/recommend.js rename to src/routes/v1/recommend.ts index 6601d48..16a3032 100644 --- a/src/routes/v1/recommend.js +++ b/src/routes/v1/recommend.ts @@ -4,59 +4,64 @@ * manipuating data from Elastic Search. */ -const router = require("express").Router(); -const { default: Elasticsearch } = require("../../library/elasticsearch"); -const { ErrorHandler } = require("../../library/error"); - -// internal modules -const mimetypes = require("../../config/mimetypes"); - +import { + IConfiguration, + IElasticsearchHit, + ISearch, + IQueryElement +} from "../../Interfaces"; + +import { Router, Request, Response, NextFunction } from "express"; // validating the query parameters -const { query, body, param } = require("express-validator"); - +import { query } from "express-validator"; // creation of the query string to help the user navigate through -const querystring = require("querystring"); - -/** - * Returns the general material type. - * @param {String} mimetype - The document mimetype. - * @returns {String|Null} The material type. - */ -function materialType(mimetype) { - for (let type in mimetypes) { - if (mimetypes[type].includes(mimetype)) { - return type; - } - } - return null; -} - -function materialFormat(hit, { wikipedia, wikipedia_limit }) { +import * as querystring from "querystring"; +// add error handling functionality +import { ErrorHandler } from "../../library/error"; +// import elasticsearch module +import Elasticsearch from "../../library/elasticsearch"; + +// initialize the express router +const router = Router(); + +// format the material +function materialFormat( + hit: IElasticsearchHit, + wikipedia?: boolean, + wikipedia_limit?: number +) { return { weight: hit._score, + material_id: hit._source.material_id, title: hit._source.title, description: hit._source.description, creation_date: hit._source.creation_date, retrieved_date: hit._source.retrieved_date, type: hit._source.type, mimetype: hit._source.mimetype, + url: hit._source.material_url, website: hit._source.website_url, language: hit._source.language, license: hit._source.license, provider: { id: hit._source.provider_id, name: hit._source.provider_name.toLowerCase(), - domain: hit._source.provider_url, - } + domain: hit._source.provider_url + }, + content_ids: hit._source.contents + ? hit._source.contents.map(content => content.content_id) + : [], + ...(wikipedia && { + wikipedia: + wikipedia_limit && wikipedia_limit > 0 + ? hit._source.wikipedia.slice(0, wikipedia_limit) + : hit._source.wikipedia + }) }; } -/** - * @description Assign the elasticsearch API routes. - * @param {Object} config - The configuration object. - * @returns {Object} The search router. - */ -module.exports = (config) => { +// assign the recommendation API routes +export default (config: IConfiguration) => { // set the default parameters const DEFAULT_LIMIT = 20; const MAX_LIMIT = 100; @@ -65,60 +70,47 @@ module.exports = (config) => { // esablish connection with elasticsearch const es = new Elasticsearch(config.elasticsearch); - let setLanguages = []; - es.search("oer_materials", { - size: 0, - aggregations: { - languages: { - terms: { field: "language" } - }, - } - }).then(({ aggregations }) => { - setLanguages = aggregations.languages.buckets - .map((obj) => obj.key); - }); - /** * @api {GET} /api/v1/oer_materials Search through the OER materials * @apiVersion 1.0.0 * @apiName searchAPI * @apiGroup search */ + // TODO: must specify the correct material type router.get("/recommend/bundles", [ query("text").trim(), query("url").trim(), query("types").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase() : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase() : null)), query("licenses").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("languages").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("content_languages").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",") : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",") : null)), query("provider_ids").optional().trim() - .customSanitizer((value) => (value && value.length ? value.toLowerCase().split(",").map((id) => parseInt(id, 10)) : null)), + .customSanitizer((value: string) => (value && value.length ? value.toLowerCase().split(",").map(id => parseInt(id, 10)) : null)), query("wikipedia").optional().toBoolean(), query("wikipedia_limit").optional().toInt(), query("limit").optional().toInt(), - query("page").optional().toInt(), - ], async (req, res, next) => { - + query("page").optional().toInt() + ], async (req: Request, res: Response, next: NextFunction) => { // extract the appropriate query parameters - let { - query: { - text, - url, - types, - languages, - content_languages, - provider_ids, - licenses, - wikipedia, - wikipedia_limit, - limit, - page, - } - } = req; + const requestQuery: ISearch = req.query; + // extract the appropriate query parameters + const { + text, + url, + types, + languages, + content_languages, + provider_ids, + licenses, + wikipedia, + wikipedia_limit, + limit: queryLimit, + page: queryPage + } = requestQuery; if (!text && !url) { return res.status(400).json({ @@ -138,9 +130,8 @@ module.exports = (config) => { } }; - let wikiConcepts; - let materialURLs; - let preferedLangs; + let wikiConcepts: [string, number][]; + let materialURLs: string[]; try { // get the search results from elasticsearch const results = await es.search("oer_materials", materialQuery); @@ -149,24 +140,23 @@ module.exports = (config) => { return res.redirect(`/api/v1/oer_materials?${queryParams}`); } - const viewedMaterials = results.hits.hits; - const viewedLangauges = viewedMaterials.map((hit) => hit._source.language); - materialURLs = viewedMaterials.map((hit) => hit._source.material_url); - preferedLangs = setLanguages.filter((lang) => !viewedLangauges.includes(lang)); + const viewedMaterials: IElasticsearchHit[] = results.hits.hits; + materialURLs = viewedMaterials.map(hit => hit._source.material_url); // return res.json(materialURLs); - wikiConcepts = viewedMaterials - .map((hit) => hit._source.wikipedia.slice(0, 30).map((wiki) => wiki.sec_name)) + wikiConcepts = Object.entries(viewedMaterials + .map(hit => hit._source.wikipedia.slice(0, 30).map(wiki => wiki.sec_name)) .reduce((prev, curr) => prev.concat(curr), []) .reduce((prev, curr) => { if (!prev[curr]) { - prev[curr] = 1; + prev[curr] = 0; } prev[curr] += 1; return prev; - }, {}); + }, {})); const startSlice = wikiConcepts.length > 2 ? 2 : 0; - wikiConcepts = Object.entries(wikiConcepts).sort((a, b) => b[1] - a[1]) + wikiConcepts = wikiConcepts + .sort((a, b) => b[1] - a[1]) .slice(startSlice, 20); } catch (error) { return next(new ErrorHandler(500, "Internal server error")); @@ -177,27 +167,34 @@ module.exports = (config) => { // ------------------------------------ // set default pagination values - if (!limit) { - limit = DEFAULT_LIMIT; - } else if (limit <= 0) { - limit = DEFAULT_LIMIT; - } else if (limit >= MAX_LIMIT) { - limit = MAX_LIMIT; - } + // which part of the materials do we want to query + const limit: number = !queryLimit + ? DEFAULT_LIMIT + : queryLimit <= 0 + ? DEFAULT_LIMIT + : queryLimit >= MAX_LIMIT + ? DEFAULT_LIMIT + : queryLimit; + + const page: number = !queryPage + ? DEFAULT_PAGE + : queryPage; + + const size = limit; + const from = (page - 1) * size; + req.query.limit = limit; - if (!page) { - page = DEFAULT_PAGE; - req.query.page = page; - } + req.query.page = page; // ------------------------------------ // Set query parameters // ------------------------------------ // set the nested must conditions for the "contents" attribute - const nestedContentsMust = [{ + const nestedContentsMust: IQueryElement[] = [{ term: { "contents.extension": "plain" } }]; + if (content_languages) { nestedContentsMust.push({ terms: { "contents.language": content_languages } @@ -209,24 +206,27 @@ module.exports = (config) => { // ------------------------------------ // get the filter parameters (type and language) - let typegroup; - let filetypes; + let typegroup: string; + let filetypes: string; if (types && ["all", "text", "video", "audio"].includes(types)) { typegroup = types === "all" ? null : types; } else if (types && types.split(",").length > 0) { - filetypes = types.split(",").map((t) => `.*\.${t.trim()}`).join("|"); + filetypes = types + .split(",") + .map(t => `.*\.${t.trim()}`) + .join("|"); } // add the must not filter conditions - const filtersMustNot = []; + const filtersMustNot: IQueryElement[] = []; if (materialURLs) { filtersMustNot.push({ terms: { material_url: materialURLs } - }) + }); } // add the filter conditions for the regex - const filtersMust = []; + const filtersMust: IQueryElement[] = []; if (filetypes) { filtersMust.push({ regexp: { material_url: filetypes } @@ -263,11 +263,7 @@ module.exports = (config) => { } // check if we need to filter the documents - const filterFlag = filtersMust.length || filtersMustNot.length; - - // which part of the materials do we want to query - const size = limit; - const from = (page - 1) * size; + const filterFlag = filtersMust.length > 0 || filtersMustNot.length > 0; // ------------------------------------ // Set the elasticsearch query body @@ -287,7 +283,7 @@ module.exports = (config) => { }, query: { bool: { - should: wikiConcepts.map((wiki) => ({ + should: wikiConcepts.map(wiki => ({ nested: { path: "wikipedia", query: { @@ -296,18 +292,18 @@ module.exports = (config) => { query: wiki[0], boost: wiki[1] / materialURLs.length } - }, + } } } })), - ...filterFlag && { + ...(filterFlag && { filter: { bool: { - ...filtersMust && { must: filtersMust }, - ...filtersMustNot && { must_not: filtersMustNot } + ...(filtersMust && { must: filtersMust }), + ...(filtersMustNot && { must_not: filtersMustNot }) } } - } + }) } }, collapse: { @@ -336,27 +332,36 @@ module.exports = (config) => { const results = await es.search("oer_materials", esQuery); // return res.json(results); // format the output before sending - const output = results.hits.hits.map((hit) => materialFormat(hit, { wikipedia, wikipedia_limit })); + const output = results.hits.hits.map((hit: IElasticsearchHit) => + materialFormat(hit, wikipedia, wikipedia_limit) + ); // prepare the parameters for the previous query const prevQuery = { ...req.query, - ...page && { page: page - 1 }, + ...(page && { page: page - 1 }) }; // prepare the parameters for the next query const nextQuery = { ...req.query, - ...page && { page: page + 1 }, + ...(page && { page: page + 1 }) }; - const BASE_URL = "https://platform.x5gon.org/api/v1/recommend/materials"; + const BASE_URL = + "https://platform.x5gon.org/api/v1/recommend/materials"; // prepare the metadata used to navigate through the search const totalHits = results.hits.total.value; const totalPages = Math.ceil(results.hits.total.value / size); - const prevPage = page - 1 > 0 ? `${BASE_URL}?${querystring.stringify(prevQuery)}` : null; - const nextPage = totalPages >= page + 1 ? `${BASE_URL}?${querystring.stringify(nextQuery)}` : null; - results.aggregations.providers.buckets.forEach((provider) => { + const prevPage = + page - 1 > 0 + ? `${BASE_URL}?${querystring.stringify(prevQuery)}` + : null; + const nextPage = + totalPages >= page + 1 + ? `${BASE_URL}?${querystring.stringify(nextQuery)}` + : null; + results.aggregations.providers.buckets.forEach(provider => { provider.key = provider.key.toLowerCase(); }); @@ -380,7 +385,8 @@ module.exports = (config) => { } catch (error) { return next(new ErrorHandler(500, "Internal server error")); } - }); + } + ); // return the router return router; diff --git a/tsconfig.json b/tsconfig.json index 21e0e38..7d440e1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,6 +7,7 @@ "lib": ["es2017"], "noImplicitReturns": true, "noFallthroughCasesInSwitch": true, + "resolveJsonModule": true, "alwaysStrict": true, "noUnusedLocals": true, "pretty": false, diff --git a/tslint.json b/tslint.json index ac2c48d..39a8943 100644 --- a/tslint.json +++ b/tslint.json @@ -19,9 +19,8 @@ }, "linterOptions": { "exclude": [ - "public/**/*.js", - "config/**/*.js", - "node_modules/**/*.ts" + "node_modules/**/*.ts", + "src/load/*.js" ] } }