Backend: Add metrics & text index

Signed-off-by: Fred Boniface <fred@fjla.uk>
This commit is contained in:
Fred Boniface 2023-02-02 14:36:42 +00:00
parent d381f85ad0
commit 20dbac3ebc
7 changed files with 68 additions and 9 deletions

View File

@ -5,6 +5,7 @@
* Alerts box should not be clickable, bar should be. * Alerts box should not be clickable, bar should be.
* Enable text search for `locationName` on find-code page. * Enable text search for `locationName` on find-code page.
* Responsive text sizes for boards. * Responsive text sizes for boards.
* Build metrics page
### In Progress: ### In Progress:
@ -20,11 +21,13 @@
* Add Gitea Issue API * Add Gitea Issue API
* Issue page: Check for success and then redirect to /. * Issue page: Check for success and then redirect to /.
* Add success test for Gitea Issue API and send the result onto the client. * Add success test for Gitea Issue API and send the result onto the client.
* DB Indexes:
- "stations": 3ALPHA, STANOX, TIPLOC
- "corpus": 3ALPHA, NLC
## Backend: ## Backend:
* DB Indexes: * DB Indexes:.
- "stations": 3ALPHA, STANOX, TIPLOC - "corpus": NLCDESC(TEXT)
- "corpus": 3ALPHA, NLC, NLCDESC(TEXT)
* Rewrite sanitizing functions to remove external dependancy. * Rewrite sanitizing functions to remove external dependancy.
* Undo changed to make everything an array - frontend code to handle this. * Undo changed to make everything an array - frontend code to handle this.

View File

@ -18,7 +18,17 @@ async function getCorpus(req, res, next){
} }
} }
async function hits(req, res, next) {
try {
res.json(await list.hits())
} catch (err) {
console.error(`Controller Error`, err);
next(err);
}
}
module.exports = { module.exports = {
getStations, getStations,
getCorpus getCorpus,
hits
} }

View File

@ -16,5 +16,6 @@ const listController = require('../controllers/list.controllers');
router.get('/stations', listController.getStations); router.get('/stations', listController.getStations);
router.get('/corpus', listController.getCorpus); router.get('/corpus', listController.getCorpus);
router.get('/hits', listController.hits)
module.exports = router; module.exports = router;

View File

@ -78,19 +78,53 @@ async function query(collection, query){
return (await qcursor.toArray()); return (await qcursor.toArray());
} }
async function ensureIndex(col, field) { async function ensureIndex(col, field, text) {
await client.connect(); await client.connect();
if (!text) {
log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`) log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`)
let res = await db.createIndex(col, field); db.createIndex(col, field);
} else {
log.out(`dbAccess.ensureIndex: Creating text index in collection ${col} for field ${field}`)
let idx = {}
idx[field] = "text";
db.createIndex(col, idx);
}
log.out(`dbAccess.ensureIndex: Index created`); log.out(`dbAccess.ensureIndex: Index created`);
return; return;
} }
async function increment(target) {
await client.connect();
let col = db.collection("meta");
let update = {}
update[target] = 1
col.updateOne({target: "ext_api"}, {$inc:update})
return;
}
async function createCount() {
await client.connect();
let col = db.collection("meta");
var filter = {type: "count", target: "ext_api"};
var update = {$set:{since: new Date, type: "count", target: "ext_api"}};
var options = {upsert: true}; // If document isn't present will insert.
try {
var result = await col.updateOne(filter,update,options)
log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
log.out(`dbAccessServices.updateMeta: count meta added updated`)
} catch (err) {
log.out(`dbAccessServices.updateMeta: Unable to add count`)
log.out(err)
}
}
module.exports = { module.exports = {
putCorpus, putCorpus,
putStations, putStations,
dropCollection, dropCollection,
updateMeta, updateMeta,
query, query,
ensureIndex ensureIndex,
increment,
createCount
} }

View File

@ -11,6 +11,7 @@ const log = require('../utils/log.utils'); // Log Helper
const ldb = require('ldbs-json') const ldb = require('ldbs-json')
const util = require('../utils/ldb.utils') const util = require('../utils/ldb.utils')
const san = require('../utils/sanitizer.utils') const san = require('../utils/sanitizer.utils')
const db = require('../services/dbAccess.services')
const ldbKey = process.env.OWL_LDB_KEY const ldbKey = process.env.OWL_LDB_KEY
const ldbsvKey = process.env.OWL_LDB_SVKEY const ldbsvKey = process.env.OWL_LDB_SVKEY
@ -22,6 +23,7 @@ async function get(body, id){
var crs = obj[0]['3ALPHA']; var crs = obj[0]['3ALPHA'];
log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`); log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`);
var data = await arrDepBoard(crs); var data = await arrDepBoard(crs);
db.increment("ldbws") // Need to add creation of this document to the database. >> {type:"count",counting:"api_hit",target:"ldbws",since:"DATE"}
} catch (err) { } catch (err) {
log.out(`ldbService.get: Error, Unable to find CRS: ${err}`) log.out(`ldbService.get: Error, Unable to find CRS: ${err}`)
var data = {ERROR:'NOT_FOUND',description:'The entered station was not found. Please check and try again.'}; var data = {ERROR:'NOT_FOUND',description:'The entered station was not found. Please check and try again.'};

View File

@ -14,7 +14,14 @@ async function getCorpus(){
return out; return out;
} }
async function hits(){
var out = await db.query("meta");
log.out(`listServices.meta: fetched server meta`)
return out;
}
module.exports = { module.exports = {
getStations, getStations,
getCorpus getCorpus,
hits
} }

View File

@ -29,6 +29,7 @@ async function init(){
} }
} }
indexes(); indexes();
dbAccess.createCount();
} }
async function check(coll){ async function check(coll){
@ -81,6 +82,7 @@ async function indexes() {
dbAccess.ensureIndex("stations", "3ALPHA"); dbAccess.ensureIndex("stations", "3ALPHA");
dbAccess.ensureIndex("stations", "STANOX"); dbAccess.ensureIndex("stations", "STANOX");
dbAccess.ensureIndex("stations", "TIPLOC"); dbAccess.ensureIndex("stations", "TIPLOC");
dbAccess.ensureIndex("corpus", "NLCDESC", "text")
} }
module.exports = { module.exports = {