FullStack: Add metrics API

Signed-off-by: Fred Boniface <fred@fjla.uk>

UpNext.md

@@ -3,9 +3,9 @@
 ## Frontend:
 
 * Alerts box should not be clickable, bar should be.
-* Issue page: Captcha.
 * Enable text search for `locationName` on find-code page.
 * Responsive text sizes for boards.
+* Build metrics page
 
 ### In Progress:
 
@@ -21,11 +21,13 @@
 * Add Gitea Issue API
 * Issue page: Check for success and then redirect to /.
 * Add success test for Gitea Issue API and send the result onto the client.
+* DB Indexes:
+    - "stations": 3ALPHA, STANOX, TIPLOC
+    - "corpus": 3ALPHA, NLC
 
 ## Backend:
 
-* DB Indexes:
+* DB Indexes:.
-    - "stations": 3ALPHA, STANOX, TIPLOC
+    - "corpus": NLCDESC(TEXT)
-    - "corpus": 3ALPHA, NLC, NLCDESC(TEXT)
 * Rewrite sanitizing functions to remove external dependancy.
 * Undo changed to make everything an array - frontend code to handle this.

src/controllers/list.controllers.js

@@ -18,7 +18,17 @@ async function getCorpus(req, res, next){
     }
 }
 
+async function hits(req, res, next) {
+    try {
+        res.json(await list.hits())
+    } catch (err) {
+        console.error(`Controller Error`, err);
+        next(err);
+    }
+}
+
 module.exports = {
     getStations,
-    getCorpus
+    getCorpus,
+    hits
 }

src/routes/list.routes.js

@@ -16,5 +16,6 @@ const listController = require('../controllers/list.controllers');
 
 router.get('/stations', listController.getStations);
 router.get('/corpus', listController.getCorpus);
+router.get('/hits', listController.hits)
 
 module.exports = router;

src/services/dbAccess.services.js

@@ -78,10 +78,53 @@ async function query(collection, query){
     return (await qcursor.toArray());
 }
 
+async function ensureIndex(col, field, text) {
+    await client.connect();
+    if (!text) {
+        log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`)
+        db.createIndex(col, field);
+    } else {
+        log.out(`dbAccess.ensureIndex: Creating text index in collection ${col} for field ${field}`)
+        let idx = {}
+        idx[field] = "text";
+        db.createIndex(col, idx);
+    }
+    log.out(`dbAccess.ensureIndex: Index created`);
+    return;
+}
+
+async function increment(target) {
+    await client.connect();
+    let col = db.collection("meta");
+    let update = {}
+    update[target] = 1
+    col.updateOne({target: "ext_api"}, {$inc:update})
+    return;
+}
+
+async function createCount() {
+    await client.connect();
+    let col = db.collection("meta");
+    var filter = {type: "count", target: "ext_api"};
+    var update = {$set:{since: new Date, type: "count", target: "ext_api"}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await col.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: count meta added updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to add count`)
+        log.out(err)
+    }
+}
+
 module.exports = {
     putCorpus,
     putStations,
     dropCollection,
     updateMeta,
-    query
+    query,
+    ensureIndex,
+    increment,
+    createCount
 }

src/services/ldb.services.js

@@ -11,6 +11,7 @@ const log  = require('../utils/log.utils');          // Log Helper
 const ldb  = require('ldbs-json')
 const util = require('../utils/ldb.utils')
 const san  = require('../utils/sanitizer.utils')
+const db   = require('../services/dbAccess.services')
 
 const ldbKey   = process.env.OWL_LDB_KEY
 const ldbsvKey = process.env.OWL_LDB_SVKEY
@@ -22,6 +23,7 @@ async function get(body, id){
         var crs = obj[0]['3ALPHA'];
         log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`);
         var data = await arrDepBoard(crs);
+        db.increment("ldbws") // Need to add creation of this document to the database. >> {type:"count",counting:"api_hit",target:"ldbws",since:"DATE"}
     } catch (err) {
         log.out(`ldbService.get: Error, Unable to find CRS: ${err}`)
         var data = {ERROR:'NOT_FOUND',description:'The entered station was not found.  Please check and try again.'};

src/services/list.services.js

@@ -1,6 +1,6 @@
 const log    = require('../utils/log.utils');          // Log Helper
 const db     = require('../services/dbAccess.services')
-const corpus = require('../services/corpus.services');
+const os     = require('os')
 
 async function getStations(){
     var out = await db.query("stations")
@@ -14,7 +14,17 @@ async function getCorpus(){
     return out;
 }
 
+async function hits(){
+    var dat = await db.query("meta", {target: "ext_api"});
+    log.out(`listServices.meta: fetched server meta`)
+    let out = {}
+    out.host = os.hostname()
+    out.dat = dat
+    return out;
+}
+
 module.exports = {
     getStations,
-    getCorpus
+    getCorpus,
+    hits
 }

src/utils/dbinit.utils.js

@@ -12,7 +12,7 @@ async function init(){
     var status = await check('corpus');
     if (status == "not_ready") {
         try {
-            build("corpus")
+            await build("corpus")
         } catch (err) {
             log.out("dbInitUtils.init: Error building corpus database")
             log.out(err)
@@ -22,12 +22,14 @@ async function init(){
     var status = await check('stations')
     if (status == "not_ready") {
         try {
-            build("stations")
+            await build("stations")
         } catch (err) {
             log.out("dbInitUtils.init: Error building stations database")
             log.out(err)
         }
     }
+    indexes();
+    dbAccess.createCount();
 }
 
 async function check(coll){
@@ -61,7 +63,7 @@ async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
         await dbAccess.dropCollection("corpus");
         dbAccess.putCorpus(corpusAll);
 
-        log.out(`dbInitUtils.build: Updating corpus meta`)
+        log.out(`dbInitUtils.build: Updating corpus meta`);
         dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
     }
     if (db === "stations") {
@@ -69,11 +71,20 @@ async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
         var corpusSubset = await corpus.subset(corpusAll);
         dbAccess.putStations(corpusSubset);
 
-        log.out(`dbInitUtils.build: Updating stations meta`)
+        log.out(`dbInitUtils.build: Updating stations meta`);
         dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
     }
 }
 
+async function indexes() {
+    dbAccess.ensureIndex("corpus", "NLC");
+    dbAccess.ensureIndex("corpus", "3ALPHA");
+    dbAccess.ensureIndex("stations", "3ALPHA");
+    dbAccess.ensureIndex("stations", "STANOX");
+    dbAccess.ensureIndex("stations", "TIPLOC");
+    dbAccess.ensureIndex("corpus", "NLCDESC", "text")
+}
+
 module.exports = {
     init
 }

static/js/stat.js

@@ -0,0 +1,20 @@
+init();
+
+async function init() {
+    display(await get())
+}
+
+async function get() {
+    var url = `${window.location.origin}/api/v1/list/hits`;
+    var resp = await fetch(url);
+    return await resp.json();
+}
+
+async function display(data) {
+    document.getElementById('server_host').textContent = `HOST: ${data.host}`;
+    let dat = data.dat[0]
+    console.log(JSON.stringify(dat))
+    document.getElementById('time').textContent = `COUNTER START TIME: ${dat.since}`;
+    document.getElementById('ldbws').textContent = dat.ldbws || "0";
+    document.getElementById('ldbsvws').textContent = dat.ldbsvws || "0";
+}

static/stat.html

@@ -0,0 +1,18 @@
+<html lang="en">
+    <head>
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>OwlBoard - STATS</title>
+        <script src="./js/stat.js"></script>
+    </head>
+    <body>
+        <h1>OwlBoard Server Stats</h1>
+        <h2 id="server_host"></h2>
+        <p id="time"></p>
+        <h3>LDBWS API Hits: <span id="ldbws"></span></h3>
+        <h3>LDBSVWS API Hits: <span id="ldbsvws"></span></h3>
+        <br><br>
+        <p>Multiple servers can be attached to one database.  All servers sharing this servers DB are represented in these statistics, 
+           responses that do not lead to an upstream API hit are not counted.
+        </p>
+    </body>
+</html>