Initial Push (v0.0.2)

Signed-off-by: Fred Boniface <fred@fjla.uk>

src/configs/domains.configs.js

@@ -0,0 +1,10 @@
+module.exports = valid
+
+const valid = [
+    "owlboard.co.uk",
+    "fjla.uk",
+    "gwr.com",
+    "swrailway.com",
+    "firstrail.com",
+    "networkrail.co.uk"
+]

src/configs/version.configs.js

@@ -0,0 +1,6 @@
+const version = {
+    api: ["/api/v1/",],
+    app: "0.0.2"
+};
+
+module.exports = version;

src/controllers/find.controllers.js

@@ -0,0 +1,58 @@
+const find = require('../services/find.services');
+
+async function findName(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await find.name(id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+async function findCrs(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await find.crs(id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+async function findNlc(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await find.nlc(id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+async function findTiploc(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await find.tiploc(id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+async function findStanox(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await find.stanox(id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+module.exports = {
+    findName,
+    findCrs,
+    findNlc,
+    findTiploc,
+    findStanox
+}

src/controllers/issue.controllers.js

@@ -0,0 +1,14 @@
+const issue = require('../services/issue.services');
+
+async function post(req, res, next){
+    try {
+        res.json(await issue.processor(req.body))
+    } catch (err) {
+        console.error(`Controller Error`, err.message);
+        next(err);
+    }
+}
+
+module.exports = {
+    post
+}

src/controllers/kube.controllers.js

@@ -0,0 +1,34 @@
+const kube = require('../services/kube.services');
+
+async function getAlive(req, res, next){
+    try {
+        var state = kube.getAlive()
+        res.status((await state).code).send((await state).state)
+    } catch (err) {
+        res.status("503").send({state: "error"})
+    }
+}
+
+async function getReady(req, res, next){
+    try {
+        res.json(await kube.getReady(req.body))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+async function getTime(req, res, next){
+    try {
+        res.json(await kube.getTime(req.body))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+module.exports = {
+    getAlive,
+    getReady,
+    getTime
+}

src/controllers/ldb.controllers.js

@@ -0,0 +1,15 @@
+const ldb = require('../services/ldb.services');
+
+async function get(req, res, next){
+    try {
+        var id = req.params.id
+        res.json(await ldb.get(req.body, id))
+    } catch (err) {
+        console.error(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
+module.exports = {
+    get
+}

src/controllers/list.controllers.js

@@ -0,0 +1,34 @@
+const list = require('../services/list.services');
+
+async function getStations(req, res, next){
+    try {
+        res.json(await list.getStations(req.body))
+    } catch (err) {
+        console.error(`Controller Error`, err.message);
+        next(err);
+    }
+}
+
+async function getCorpus(req, res, next){
+    try {
+        res.json(await list.getCorpus(req.body))
+    } catch (err) {
+        console.error(`Controller Error`, err.message);
+        next(err);
+    }
+}
+
+async function hits(req, res, next) {
+    try {
+        res.json(await list.hits())
+    } catch (err) {
+        console.error(`Controller Error`, err);
+        next(err);
+    }
+}
+
+module.exports = {
+    getStations,
+    getCorpus,
+    hits
+}

src/controllers/stats.controllers.js

@@ -0,0 +1,13 @@
+const stat = require('../services/stats.services');
+
+async function get(req, res, next) {
+    try {
+        res.json(await stat.hits())
+    } catch (err) {
+        console.error(`Controller Error`, err);
+        next(err);
+    }
+}
+
+module.exports = {
+    get}

src/routes/find.routes.js

@@ -0,0 +1,23 @@
+const express = require('express');
+const router = express.Router();
+const findController = require('../controllers/find.controllers');
+
+/* GET programming languages. */
+//router.get('/', programmingLanguagesController.get);
+  
+/* POST programming language */
+//router.post('/', programmingLanguagesController.create);
+
+/* PUT programming language */
+//router.put('/:id', programmingLanguagesController.update);
+
+/* DELETE programming language */
+//router.delete('/:id', programmingLanguagesController.remove);
+
+router.get('/name/:id', findController.findName);
+router.get('/crs/:id', findController.findCrs);
+router.get('/nlc/:id', findController.findNlc);
+router.get('/tiploc/:id', findController.findTiploc);
+router.get('/stanox/:id', findController.findStanox);
+
+module.exports = router;

src/routes/issue.routes.js

@@ -0,0 +1,7 @@
+const express = require('express');
+const router = express.Router();
+const issueController = require('../controllers/issue.controllers');
+
+router.post('/', issueController.post);
+
+module.exports = router;

src/routes/kube.routes.js

@@ -0,0 +1,9 @@
+const express = require('express');
+const router = express.Router();
+const kubeController = require('../controllers/kube.controllers');
+
+router.get('/alive', kubeController.getAlive);
+router.get('/ready', kubeController.getReady);
+router.get('/time', kubeController.getTime);
+
+module.exports = router

src/routes/ldb.routes.js

@@ -0,0 +1,19 @@
+const express = require('express');
+const router = express.Router();
+const ldbController = require('../controllers/ldb.controllers');
+
+/* GET programming languages. */
+//router.get('/', programmingLanguagesController.get);
+  
+/* POST programming language */
+//router.post('/', programmingLanguagesController.create);
+
+/* PUT programming language */
+//router.put('/:id', programmingLanguagesController.update);
+
+/* DELETE programming language */
+//router.delete('/:id', programmingLanguagesController.remove);
+
+router.get('/:id', ldbController.get);
+
+module.exports = router;

src/routes/list.routes.js

@@ -0,0 +1,20 @@
+const express = require('express');
+const router = express.Router();
+const listController = require('../controllers/list.controllers');
+
+/* GET programming languages. */
+//router.get('/', programmingLanguagesController.get);
+  
+/* POST programming language */
+//router.post('/', programmingLanguagesController.create);
+
+/* PUT programming language */
+//router.put('/:id', programmingLanguagesController.update);
+
+/* DELETE programming language */
+//router.delete('/:id', programmingLanguagesController.remove);
+
+router.get('/stations', listController.getStations);
+router.get('/corpus', listController.getCorpus);
+
+module.exports = router;

src/routes/stats.routes.js

@@ -0,0 +1,8 @@
+const express = require('express');
+const router = express.Router();
+const statsController = require('../controllers/stats.controllers');
+
+
+router.get('/', statsController.get);
+
+module.exports = router;

src/services/corpus.services.js

@@ -0,0 +1,73 @@
+// Get CORPUS data from Network Rail and format the data for OwlBoard
+
+// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
+
+// FUNCTIONS/
+//  initSubset()  : Exported: Uses the internal functions to return a clean CORPUS object.
+//  initAll()     : Exported: Uses the internal functions to return a full CORPUS object.
+//  get()         : Get the CORPUS data from Network Rail as a gzip file.
+//  extract()     : Extract the CORPUS JSON file from the GZIP file.
+//  clean()       : Cleans the CORPUS data, removing unneccesary non-stations from the data.
+
+const log    = require('../utils/log.utils');          // Log Helper
+
+const axios  = require('axios')
+const gz     = require('node-gzip')
+
+const corpusUser = process.env.OWL_LDB_CORPUSUSER
+const corpusPass = process.env.OWL_LDB_CORPUSPASS
+
+async function subset(allCorpus) {
+    return (await clean(allCorpus))
+}
+
+async function get() {
+    var gzipData = await fetch()
+    return (await extract(gzipData))
+}
+
+async function fetch() {
+    log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
+    authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
+    const url = 'https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
+    const options = {
+        method: 'get',
+        timeout: 20000,
+        headers: {'Authorization': `Basic ${authHead}`},
+        responseType: 'arraybuffer'
+    }
+    try {
+        var { data } = await axios.get(url, options)
+        log.out("corpus.fetch: CORPUS Data fetched")
+    } catch (error) {
+        log.out("corpus.fetch: Error fetching CORPUS")
+        log.out(error)
+    }
+    return data
+}
+
+async function extract(input) {
+    log.out(`corpus.extract: Extracting CORPUS archive`)
+    var raw = await gz.ungzip(input)
+    var obj = await JSON.parse(raw)
+    return (obj.TIPLOCDATA)
+}
+
+async function clean(input) {
+    log.out(`corpus.clean: Removing non-stations from CORPUS data`)
+    let clean = [];
+    for (const element of input) {
+        if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
+            delete(element.UIC);
+            delete(element.NLCDESC16);
+            delete(element.NLC);
+            clean.push(element);
+        }
+    }
+    return clean;
+}
+
+module.exports = {
+    get,
+    subset
+}

src/services/dbAccess.services.js

@@ -0,0 +1,131 @@
+const log      = require('../utils/log.utils');          // Log Helper
+
+const dbUser = process.env.OWL_DB_USER || "owl"
+const dbPass = process.env.OWL_DB_PASS || "twittwoo"
+const dbName = process.env.OWL_DB_NAME || "owlboard"
+const dbPort = process.env.OWL_DB_PORT || 27017
+const dbHost = process.env.OWL_DB_HOST || "localhost"
+const uri = `mongodb://${dbUser}:${dbPass}@${dbHost}:${dbPort}`;
+
+const { MongoClient } = require('mongodb');
+
+const client = new MongoClient(uri);
+const db = client.db(dbName);
+
+async function dropCollection(coll){
+    await client.connect();
+
+    // check if collection contains any documents, if it doesn't, it is either empty or non-existent - it doesn't need dropping.
+    var collection = db.collection(coll);
+    var count = await collection.countDocuments();
+    log.out(`DbAccess.dropCollection: Collection '${coll}' contains ${count} documents`)
+    if (count == 0) {
+        log.out(`DbAccess.dropCollection: Collection '${coll}' is empty.  Do not need to drop`)
+    } else {
+        log.out(`DbAccess.dropCollection: dropping collection: '${coll}'`)
+        db.dropCollection(coll);
+        log.out(`DbAccess.dropCollection: dropped collection: '${coll}'`)
+    }
+}
+
+async function putCorpus(data){
+    log.out("DbAccess.putCorpus: Uploading CORPUS data to database")
+    await client.connect();
+    try {
+        var coll = db.collection("corpus");
+        await coll.insertMany(data);
+    } catch (error) {
+        log.out("DbAccess.putCorpus: Error uploading Corpus data to database")
+        log.out(error)
+    }
+};
+
+async function putStations(data){
+    log.out("DbAccess.putStations: Uploading Stations data to database")
+    await client.connect();
+    try {
+        var coll = db.collection("stations");
+        coll.insertMany(data);
+    } catch (error) {
+        log.out("DbAccess.putStations: Error uploading Stations data to database")
+        log.out(error)
+    }
+};
+
+async function updateMeta(type, target, unixTime){
+    await client.connect();
+    var coll = db.collection("meta");
+    var filter = {type: type, target: target};
+    var update = {$set:{updated: unixTime}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await coll.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
+        log.out(err)
+    }
+}
+
+async function query(collection, query){
+    await client.connect();
+    log.out(`dbAccess.query: Connecting to collection: '${collection}'`)
+    var qcoll = db.collection(collection);
+    var qcursor = qcoll.find(query)
+    qcursor.project({_id: 0})
+    log.out(`dbAccess.query: Running Query: ${JSON.stringify(query)}`)
+    increment(collection)
+    return (await qcursor.toArray());
+}
+
+async function ensureIndex(col, field, text) {
+    await client.connect();
+    if (!text) {
+        log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`)
+        db.createIndex(col, field);
+    } else {
+        log.out(`dbAccess.ensureIndex: Creating text index in collection ${col} for field ${field}`)
+        let idx = {}
+        idx[field] = "text";
+        db.createIndex(col, idx);
+    }
+    log.out(`dbAccess.ensureIndex: Index created`);
+    return;
+}
+
+async function increment(target) {
+    await client.connect();
+    let col = db.collection("meta");
+    let update = {}
+    update[target] = 1
+    col.updateOne({target: "counters"}, {$inc:update})
+    return;
+}
+
+async function createCount() {
+    await client.connect();
+    let col = db.collection("meta");
+    var filter = {type: "count", target: "counters"};
+    var update = {$set:{/*since: new Date,*/ type: "count", target: "counters"}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await col.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: count meta added updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to add count`)
+        log.out(err)
+    }
+}
+
+module.exports = {
+    putCorpus,
+    putStations,
+    dropCollection,
+    updateMeta,
+    query,
+    ensureIndex,
+    increment,
+    createCount
+}

src/services/find.services.js

@@ -0,0 +1,59 @@
+// Parse and return a find request
+
+const log = require('../utils/log.utils');          // Log Helper
+const db  = require('../services/dbAccess.services');
+const san = require('../utils/sanitizer.utils')
+
+// DB Query: query(collection, query)
+
+// Define collection as all queries are for the "corpus" collection.
+const col = "corpus"
+
+async function name(id){
+    log.out(`findServices.name: Finding station name: ${id}`)
+    var name = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {NLCDESC: name}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function crs(id){
+    log.out(`findServices.crs: Finding crs: ${id}`)
+    var crs = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {'3ALPHA': crs}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function nlc(id){
+    log.out(`findServices.nlc: Finding nlc: ${id}`)
+    var nlc = san.cleanApiEndpointNum(id)
+    query = {NLC: parseInt(nlc)}
+    log.out(`findServices.nlc: NLC Converted to int: ${query}`)
+    var data = await db.query(col,query)
+    return data
+}
+
+async function tiploc(id){
+    log.out(`findServices.tiploc: Finding tiploc: ${id}`)
+    var tiploc = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {TIPLOC: tiploc}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function stanox(id){
+    log.out(`findServices.stanox: Finding stanox: ${id}`)
+    var stanox = san.cleanApiEndpointNum(id)
+    query = {STANOX: String(stanox)}
+    var data = await db.query(col,query)
+    return data
+}
+
+module.exports = {
+    name,
+    crs,
+    nlc,
+    tiploc,
+    stanox
+}

src/services/issue.services.js

@@ -0,0 +1,33 @@
+const axios  = require('axios')
+const log    = require('../utils/log.utils')
+
+async function processor(data) {
+    log.out(`issueService.processor: Issue received`)
+    let out = {}
+    out.title = data.subject.replace(/<[^>]+>|[\*\$]/g, '');
+    out.body = data.msg.replace(/<[^>]+>|[\*\$]/g, '')
+    sendToGitea(out);
+}
+
+async function sendToGitea(body) {
+    let key = process.env.OWL_GIT_ISSUEBOT
+    let url = process.env.OWL_GIT_APIENDPOINT
+    let opts = {
+        headers: {
+            Authorization: key
+        }
+    }
+    var res = await axios.post(url, body, opts)
+    // Need to read the output from the POST and pass the result upwards to the client.
+    if (res.status == 201) {
+        log.out("issueService.sendToGitea: Issue sent to Gitea")
+        return {status: res.status,message:"issue created"}
+    } else {
+        log.out("issueService.sendToGitea: Failed to send issue to Gitea")
+        return {status: res.status,message:"issue not created"}
+    }
+}
+
+module.exports = {
+    processor
+}

src/services/kube.services.js

@@ -0,0 +1,20 @@
+async function getAlive(){
+    log.out(`kubeServices.getAlive: alive hook checked`)
+    return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
+}
+
+async function getReady(){
+    log.out(`kubeServices.getReady: ready hook checked`)
+    return "not_implemented";
+};
+
+async function getTime(){
+    var now = new Date()
+    return {responseGenerated: now}
+}
+
+module.exports = {
+    getAlive,
+    getReady,
+    getTime
+}

src/services/ldb.services.js

@@ -0,0 +1,52 @@
+// Parse and return an LDB Request
+
+// FUNCTIONS
+//  post(body, id): Exported:
+            // body: [req.body from controller]
+            // id  : [req.params.id from controller - this is expected to be CRS or TIPLOC]
+
+//  convertTiploc(TIPLOC)  : Exported: Looks up CRS, Name & STANOX for Tiploc
+
+const log  = require('../utils/log.utils');          // Log Helper
+const ldb  = require('ldbs-json')
+const util = require('../utils/ldb.utils')
+const san  = require('../utils/sanitizer.utils')
+const db   = require('../services/dbAccess.services')
+
+const ldbKey   = process.env.OWL_LDB_KEY
+const ldbsvKey = process.env.OWL_LDB_SVKEY
+
+async function get(body, id){
+    var cleanId = san.cleanApiEndpointTxt(id);
+    var obj = await util.checkCrs(cleanId);
+    try {
+        var crs = obj[0]['3ALPHA'];
+        log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`);
+        var data = await arrDepBoard(crs);
+        db.increment("ldbws") // Need to add creation of this document to the database. >> {type:"count",counting:"api_hit",target:"ldbws",since:"DATE"}
+    } catch (err) {
+        log.out(`ldbService.get: Error, Unable to find CRS: ${err}`)
+        var data = {ERROR:'NOT_FOUND',description:'The entered station was not found.  Please check and try again.'};
+    }
+    return data;
+}
+
+async function arrDepBoard(CRS){
+    log.out(`ldbService.arrDepBoard: Trying to fetch ArrDep Board for ${CRS}`)
+    try {
+        var options = {
+            numRows: 10,
+            crs: CRS.toUpperCase()
+        }
+        var api = new ldb(ldbKey,false)
+        var reply = await api.call("GetArrDepBoardWithDetails",options)
+        return reply
+    } catch (err) {
+        log.out(`ldbService.arrDepBoard: Lookup Failed for: ${CRS}`)
+        return {GetStationBoardResult: "not available", Reason: `The CRS code ${CRS} is not valid`, Why: `Sometimes a station will have more than one CRS - for example Filton Abbey Wood has FIT and FAW however schedules are only available when looking up with FIT - this is how the National Rail Enquiries systems work.`};
+    }
+};
+
+module.exports = {
+    get
+}

src/services/list.services.js

@@ -0,0 +1,20 @@
+const log    = require('../utils/log.utils');          // Log Helper
+const db     = require('../services/dbAccess.services')
+const os     = require('os')
+
+async function getStations(){
+    var out = await db.query("stations")
+    log.out(`listServices.getStations: fetched stations list`)
+    return out;
+}
+
+async function getCorpus(){
+    var out = await db.query("corpus")
+    log.out(`listServices.getCorpus: fetched CORPUS list`)
+    return out;
+}
+
+module.exports = {
+    getStations,
+    getCorpus
+}

src/services/stats.services.js

@@ -0,0 +1,16 @@
+const log    = require('../utils/log.utils');          // Log Helper
+const db     = require('../services/dbAccess.services')
+const os     = require('os')
+
+async function hits(){
+    var dat = await db.query("meta", {target: "counters"});
+    log.out(`listServices.meta: fetched server meta`)
+    let out = {}
+    out.host = os.hostname()
+    out.dat = dat
+    return out;
+}
+
+module.exports = {
+    hits
+}

src/utils/dbinit.utils.js

@@ -0,0 +1,90 @@
+// FUNCTIONS
+//  init()   : Exported: Uses the internal functions to initialise databases.
+//  check()  : Checks data presence and age.
+//  build()  : Builds/Rebuilds collections.
+
+const log      = require('../utils/log.utils');          // Log Helper
+const time     = require('../utils/timeConvert.utils');  // Time Helper
+const corpus   = require('../services/corpus.services');
+const dbAccess = require('../services/dbAccess.services');
+
+async function init(){
+    var status = await check('corpus');
+    if (status == "not_ready") {
+        try {
+            await build("corpus")
+        } catch (err) {
+            log.out("dbInitUtils.init: Error building corpus database")
+            log.out(err)
+        }
+    }
+
+    var status = await check('stations')
+    if (status == "not_ready") {
+        try {
+            await build("stations")
+        } catch (err) {
+            log.out("dbInitUtils.init: Error building stations database")
+            log.out(err)
+        }
+    }
+    indexes();
+    dbAccess.createCount();
+}
+
+async function check(coll){
+    log.out(`dbInitUtils.check: Checking collection '${coll}'`)
+    try {
+        var queryStr = {'type':'collection','target': coll};
+        var res = await dbAccess.query('meta',queryStr);
+        log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
+        var now = time.jsUnix(Date.now())
+        var delta = now - res['0']['updated']
+    } catch (err) {
+        log.out(`dbInitUtils.check: Unable to find out data age.  Presume stale. Error Message:`)
+        log.out(err)
+        var delta = 12096000 // Extra zero to ensure data is updated.
+    }
+
+    var maxAge = 1209600 // 14 Days
+    if (delta > maxAge) {
+        log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds.  Update pending`)
+        return "not_ready"
+    } else {
+        log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds.  Update not required`)
+        return "ready"
+    }
+}
+
+async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
+    log.out("dbInitUtils.build: Building database structure")
+    var corpusAll = await corpus.get();
+    if (db === "corpus") {
+        await dbAccess.dropCollection("corpus");
+        dbAccess.putCorpus(corpusAll);
+
+        log.out(`dbInitUtils.build: Updating corpus meta`);
+        dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
+    }
+    if (db === "stations") {
+        await dbAccess.dropCollection("stations");
+        var corpusSubset = await corpus.subset(corpusAll);
+        dbAccess.putStations(corpusSubset);
+
+        log.out(`dbInitUtils.build: Updating stations meta`);
+        dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
+    }
+}
+
+async function indexes() {
+    dbAccess.ensureIndex("corpus", "NLC");
+    dbAccess.ensureIndex("corpus", "3ALPHA");
+    dbAccess.ensureIndex("stations", "3ALPHA");
+    dbAccess.ensureIndex("stations", "STANOX");
+    dbAccess.ensureIndex("stations", "TIPLOC");
+    dbAccess.ensureIndex("corpus", "NLCDESC", "text")
+}
+
+module.exports = {
+    init
+}

src/utils/ldb.utils.js

@@ -0,0 +1,43 @@
+const log  = require('../utils/log.utils');            // Log Helper
+const db   = require('../services/dbAccess.services') // DB Access
+const san  = require('../utils/sanitizer.utils')     // Sanitiser
+
+async function checkCrs(input){
+    var INPUT = input.toUpperCase()
+    log.out(`ldbUtils.checkCrs: Building database query to find: '${INPUT}'`)
+    var query = {'$or':[{'3ALPHA':INPUT},{'TIPLOC':INPUT},{'STANOX':INPUT}]};
+    var result = await db.query("stations", query)
+    log.out(`ldbUtils.checkCrs: Query results: ${JSON.stringify(result)}`)
+    return result
+}
+
+async function cleanMessages(input){
+    var out = []
+    if (typeof input.message == "string") {
+        out.push(await san.cleanNrcc(input.message))
+    } else if (typeof input.message == "object") {
+        for(var i = 0; i < input.message.length; i++) {
+            out.push(await san.cleanNrcc(input.message[i]))
+        }
+    }
+    return out;
+}
+
+// Accepts an object but not an Array and returns it wrapped in an array.
+async function cleanServices(input){
+    var out = []
+    if (!Array.isArray(input)) {
+        log.out(`ldbUtils.cleanServices: Transforming input: ${input}`)
+        out.push(input)
+        log.out(`ldbUtils.cleanServices: Returning output: ${out}`)
+        return out;
+    } else {
+        return input;
+    }
+}
+
+module.exports = {
+    checkCrs,
+    cleanMessages,
+    cleanServices
+}

src/utils/log.utils.js

@@ -0,0 +1,8 @@
+function out(msg) {
+    var time = new Date().toISOString();
+    console.log(`${time} - ${msg}`)
+}
+
+module.exports = {
+    out
+}

src/utils/sanitizer.utils.js

@@ -0,0 +1,45 @@
+const clean = require('string-sanitizer-fix');
+const log   = require('../utils/log.utils');
+
+/*
+string.sanitize("a.bc@d efg#h"); // abcdefgh
+string.sanitize.keepSpace("a.bc@d efg#h"); // abcd efgh
+string.sanitize.keepUnicode("a.bc@d efg#hক"); // abcd efghক
+string.sanitize.addFullstop("a.bc@d efg#h"); // abcd.efgh
+string.sanitize.addUnderscore("a.bc@d efg#h"); // abcd_efgh
+string.sanitize.addDash("a.bc@d efg#h"); // abcd-efgh
+string.sanitize.removeNumber("@abcd efgh123"); // abcdefgh
+string.sanitize.keepNumber("@abcd efgh123"); // abcdefgh123
+string.addFullstop("abcd efgh"); // abcd.efgh
+string.addUnderscore("@abcd efgh"); // @abcd_efgh
+string.addDash("@abcd efgh"); // @abcd-efgh
+string.removeSpace("@abcd efgh"); // @abcdefgh
+*/
+
+function cleanApiEndpointTxt(input) {
+    var output = clean.sanitize.keepSpace(input)
+    if (output != input){
+        log.out(`sanitizerUtils.cleanApiEndpoint: WARN: Sanitizing changed string. Input = ${input}`);
+    }
+    return output
+}
+
+function cleanApiEndpointNum(input) {
+    var output = clean.sanitize.keepNumber(input)
+    if (output != input){
+        log.out(`sanitizerUtils.cleanApiEndpointNum: WARN: Sanitizing changed string. Input = ${input}`);
+    }
+    return output
+}
+
+function cleanNrcc(input) {
+    var rmNewline = input.replace(/[\n\r]/g, ""); // Remove newlines
+    var rmPara = rmNewline.replace(/<\/?p[^>]*>/g, ""); // Remove <p> & </p>
+    return rmPara;
+}
+
+module.exports = {
+    cleanApiEndpointTxt,
+    cleanApiEndpointNum,
+    cleanNrcc
+}

src/utils/timeConvert.utils.js

@@ -0,0 +1,15 @@
+function unixLocal(unix) {
+    var jsTime = unix*1000
+    var dt = new Date(jsTime)
+    return dt.toLocaleString()
+}
+
+function jsUnix(js) {
+    var preRound = js / 1000
+    return Math.round(preRound)
+}
+
+module.exports = {
+    unixLocal,
+    jsUnix,
+}

src/utils/varTest.utils.js

@@ -0,0 +1,27 @@
+// Checks that all required environment variables are present.
+// Returns True or False and offers an object detailing what is missing.
+
+async function varTest(){
+    var required = {
+        OWL_LDB_KEY: process.env.OWL_LDB_KEY,
+        OWL_LDB_CORPUSUSER: process.env.OWL_LDB_CORPUSUSER,
+        OWL_LDB_CORPUSPASS: process.env.OWL_LDB_CORPUSPASS,
+        OWL_NOT_USED: process.env.OWL_NOT_USED
+    }
+    var desired = {
+        OWL_DB_PASS: process.env.OWL_DB_PASS
+    }
+    // DO NOT LOG CREDENTIALS!!!
+
+    // Test that each of required is NOT undefined.
+      // var pass = true if all okay, false if not.
+      // Append any missing values to missing_required = []
+    // Test that each of desired is NOT undefined.
+      // Append any missing values to missing_desired = []
+
+    // Return : {pass: $pass, missong_required = $missing_required, missing_desired = $missing_desired}
+}
+
+module.exports = {
+    varTest
+}