Initial Push (v0.0.2)

Signed-off-by: Fred Boniface <fred@fjla.uk>

src/utils/dbinit.utils.js

@@ -0,0 +1,90 @@
+// FUNCTIONS
+//  init()   : Exported: Uses the internal functions to initialise databases.
+//  check()  : Checks data presence and age.
+//  build()  : Builds/Rebuilds collections.
+
+const log      = require('../utils/log.utils');          // Log Helper
+const time     = require('../utils/timeConvert.utils');  // Time Helper
+const corpus   = require('../services/corpus.services');
+const dbAccess = require('../services/dbAccess.services');
+
+async function init(){
+    var status = await check('corpus');
+    if (status == "not_ready") {
+        try {
+            await build("corpus")
+        } catch (err) {
+            log.out("dbInitUtils.init: Error building corpus database")
+            log.out(err)
+        }
+    }
+
+    var status = await check('stations')
+    if (status == "not_ready") {
+        try {
+            await build("stations")
+        } catch (err) {
+            log.out("dbInitUtils.init: Error building stations database")
+            log.out(err)
+        }
+    }
+    indexes();
+    dbAccess.createCount();
+}
+
+async function check(coll){
+    log.out(`dbInitUtils.check: Checking collection '${coll}'`)
+    try {
+        var queryStr = {'type':'collection','target': coll};
+        var res = await dbAccess.query('meta',queryStr);
+        log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
+        var now = time.jsUnix(Date.now())
+        var delta = now - res['0']['updated']
+    } catch (err) {
+        log.out(`dbInitUtils.check: Unable to find out data age.  Presume stale. Error Message:`)
+        log.out(err)
+        var delta = 12096000 // Extra zero to ensure data is updated.
+    }
+
+    var maxAge = 1209600 // 14 Days
+    if (delta > maxAge) {
+        log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds.  Update pending`)
+        return "not_ready"
+    } else {
+        log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds.  Update not required`)
+        return "ready"
+    }
+}
+
+async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
+    log.out("dbInitUtils.build: Building database structure")
+    var corpusAll = await corpus.get();
+    if (db === "corpus") {
+        await dbAccess.dropCollection("corpus");
+        dbAccess.putCorpus(corpusAll);
+
+        log.out(`dbInitUtils.build: Updating corpus meta`);
+        dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
+    }
+    if (db === "stations") {
+        await dbAccess.dropCollection("stations");
+        var corpusSubset = await corpus.subset(corpusAll);
+        dbAccess.putStations(corpusSubset);
+
+        log.out(`dbInitUtils.build: Updating stations meta`);
+        dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
+    }
+}
+
+async function indexes() {
+    dbAccess.ensureIndex("corpus", "NLC");
+    dbAccess.ensureIndex("corpus", "3ALPHA");
+    dbAccess.ensureIndex("stations", "3ALPHA");
+    dbAccess.ensureIndex("stations", "STANOX");
+    dbAccess.ensureIndex("stations", "TIPLOC");
+    dbAccess.ensureIndex("corpus", "NLCDESC", "text")
+}
+
+module.exports = {
+    init
+}

src/utils/ldb.utils.js

@@ -0,0 +1,43 @@
+const log  = require('../utils/log.utils');            // Log Helper
+const db   = require('../services/dbAccess.services') // DB Access
+const san  = require('../utils/sanitizer.utils')     // Sanitiser
+
+async function checkCrs(input){
+    var INPUT = input.toUpperCase()
+    log.out(`ldbUtils.checkCrs: Building database query to find: '${INPUT}'`)
+    var query = {'$or':[{'3ALPHA':INPUT},{'TIPLOC':INPUT},{'STANOX':INPUT}]};
+    var result = await db.query("stations", query)
+    log.out(`ldbUtils.checkCrs: Query results: ${JSON.stringify(result)}`)
+    return result
+}
+
+async function cleanMessages(input){
+    var out = []
+    if (typeof input.message == "string") {
+        out.push(await san.cleanNrcc(input.message))
+    } else if (typeof input.message == "object") {
+        for(var i = 0; i < input.message.length; i++) {
+            out.push(await san.cleanNrcc(input.message[i]))
+        }
+    }
+    return out;
+}
+
+// Accepts an object but not an Array and returns it wrapped in an array.
+async function cleanServices(input){
+    var out = []
+    if (!Array.isArray(input)) {
+        log.out(`ldbUtils.cleanServices: Transforming input: ${input}`)
+        out.push(input)
+        log.out(`ldbUtils.cleanServices: Returning output: ${out}`)
+        return out;
+    } else {
+        return input;
+    }
+}
+
+module.exports = {
+    checkCrs,
+    cleanMessages,
+    cleanServices
+}

src/utils/log.utils.js

@@ -0,0 +1,8 @@
+function out(msg) {
+    var time = new Date().toISOString();
+    console.log(`${time} - ${msg}`)
+}
+
+module.exports = {
+    out
+}

src/utils/sanitizer.utils.js

@@ -0,0 +1,45 @@
+const clean = require('string-sanitizer-fix');
+const log   = require('../utils/log.utils');
+
+/*
+string.sanitize("a.bc@d efg#h"); // abcdefgh
+string.sanitize.keepSpace("a.bc@d efg#h"); // abcd efgh
+string.sanitize.keepUnicode("a.bc@d efg#hক"); // abcd efghক
+string.sanitize.addFullstop("a.bc@d efg#h"); // abcd.efgh
+string.sanitize.addUnderscore("a.bc@d efg#h"); // abcd_efgh
+string.sanitize.addDash("a.bc@d efg#h"); // abcd-efgh
+string.sanitize.removeNumber("@abcd efgh123"); // abcdefgh
+string.sanitize.keepNumber("@abcd efgh123"); // abcdefgh123
+string.addFullstop("abcd efgh"); // abcd.efgh
+string.addUnderscore("@abcd efgh"); // @abcd_efgh
+string.addDash("@abcd efgh"); // @abcd-efgh
+string.removeSpace("@abcd efgh"); // @abcdefgh
+*/
+
+function cleanApiEndpointTxt(input) {
+    var output = clean.sanitize.keepSpace(input)
+    if (output != input){
+        log.out(`sanitizerUtils.cleanApiEndpoint: WARN: Sanitizing changed string. Input = ${input}`);
+    }
+    return output
+}
+
+function cleanApiEndpointNum(input) {
+    var output = clean.sanitize.keepNumber(input)
+    if (output != input){
+        log.out(`sanitizerUtils.cleanApiEndpointNum: WARN: Sanitizing changed string. Input = ${input}`);
+    }
+    return output
+}
+
+function cleanNrcc(input) {
+    var rmNewline = input.replace(/[\n\r]/g, ""); // Remove newlines
+    var rmPara = rmNewline.replace(/<\/?p[^>]*>/g, ""); // Remove <p> & </p>
+    return rmPara;
+}
+
+module.exports = {
+    cleanApiEndpointTxt,
+    cleanApiEndpointNum,
+    cleanNrcc
+}

src/utils/timeConvert.utils.js

@@ -0,0 +1,15 @@
+function unixLocal(unix) {
+    var jsTime = unix*1000
+    var dt = new Date(jsTime)
+    return dt.toLocaleString()
+}
+
+function jsUnix(js) {
+    var preRound = js / 1000
+    return Math.round(preRound)
+}
+
+module.exports = {
+    unixLocal,
+    jsUnix,
+}

src/utils/varTest.utils.js

@@ -0,0 +1,27 @@
+// Checks that all required environment variables are present.
+// Returns True or False and offers an object detailing what is missing.
+
+async function varTest(){
+    var required = {
+        OWL_LDB_KEY: process.env.OWL_LDB_KEY,
+        OWL_LDB_CORPUSUSER: process.env.OWL_LDB_CORPUSUSER,
+        OWL_LDB_CORPUSPASS: process.env.OWL_LDB_CORPUSPASS,
+        OWL_NOT_USED: process.env.OWL_NOT_USED
+    }
+    var desired = {
+        OWL_DB_PASS: process.env.OWL_DB_PASS
+    }
+    // DO NOT LOG CREDENTIALS!!!
+
+    // Test that each of required is NOT undefined.
+      // var pass = true if all okay, false if not.
+      // Append any missing values to missing_required = []
+    // Test that each of desired is NOT undefined.
+      // Append any missing values to missing_desired = []
+
+    // Return : {pass: $pass, missong_required = $missing_required, missing_desired = $missing_desired}
+}
+
+module.exports = {
+    varTest
+}