Initial Push (v0.0.2)

Signed-off-by: Fred Boniface <fred@fjla.uk>

src/services/corpus.services.js

@@ -0,0 +1,73 @@
+// Get CORPUS data from Network Rail and format the data for OwlBoard
+
+// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
+
+// FUNCTIONS/
+//  initSubset()  : Exported: Uses the internal functions to return a clean CORPUS object.
+//  initAll()     : Exported: Uses the internal functions to return a full CORPUS object.
+//  get()         : Get the CORPUS data from Network Rail as a gzip file.
+//  extract()     : Extract the CORPUS JSON file from the GZIP file.
+//  clean()       : Cleans the CORPUS data, removing unneccesary non-stations from the data.
+
+const log    = require('../utils/log.utils');          // Log Helper
+
+const axios  = require('axios')
+const gz     = require('node-gzip')
+
+const corpusUser = process.env.OWL_LDB_CORPUSUSER
+const corpusPass = process.env.OWL_LDB_CORPUSPASS
+
+async function subset(allCorpus) {
+    return (await clean(allCorpus))
+}
+
+async function get() {
+    var gzipData = await fetch()
+    return (await extract(gzipData))
+}
+
+async function fetch() {
+    log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
+    authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
+    const url = 'https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
+    const options = {
+        method: 'get',
+        timeout: 20000,
+        headers: {'Authorization': `Basic ${authHead}`},
+        responseType: 'arraybuffer'
+    }
+    try {
+        var { data } = await axios.get(url, options)
+        log.out("corpus.fetch: CORPUS Data fetched")
+    } catch (error) {
+        log.out("corpus.fetch: Error fetching CORPUS")
+        log.out(error)
+    }
+    return data
+}
+
+async function extract(input) {
+    log.out(`corpus.extract: Extracting CORPUS archive`)
+    var raw = await gz.ungzip(input)
+    var obj = await JSON.parse(raw)
+    return (obj.TIPLOCDATA)
+}
+
+async function clean(input) {
+    log.out(`corpus.clean: Removing non-stations from CORPUS data`)
+    let clean = [];
+    for (const element of input) {
+        if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
+            delete(element.UIC);
+            delete(element.NLCDESC16);
+            delete(element.NLC);
+            clean.push(element);
+        }
+    }
+    return clean;
+}
+
+module.exports = {
+    get,
+    subset
+}

src/services/dbAccess.services.js

@@ -0,0 +1,131 @@
+const log      = require('../utils/log.utils');          // Log Helper
+
+const dbUser = process.env.OWL_DB_USER || "owl"
+const dbPass = process.env.OWL_DB_PASS || "twittwoo"
+const dbName = process.env.OWL_DB_NAME || "owlboard"
+const dbPort = process.env.OWL_DB_PORT || 27017
+const dbHost = process.env.OWL_DB_HOST || "localhost"
+const uri = `mongodb://${dbUser}:${dbPass}@${dbHost}:${dbPort}`;
+
+const { MongoClient } = require('mongodb');
+
+const client = new MongoClient(uri);
+const db = client.db(dbName);
+
+async function dropCollection(coll){
+    await client.connect();
+
+    // check if collection contains any documents, if it doesn't, it is either empty or non-existent - it doesn't need dropping.
+    var collection = db.collection(coll);
+    var count = await collection.countDocuments();
+    log.out(`DbAccess.dropCollection: Collection '${coll}' contains ${count} documents`)
+    if (count == 0) {
+        log.out(`DbAccess.dropCollection: Collection '${coll}' is empty.  Do not need to drop`)
+    } else {
+        log.out(`DbAccess.dropCollection: dropping collection: '${coll}'`)
+        db.dropCollection(coll);
+        log.out(`DbAccess.dropCollection: dropped collection: '${coll}'`)
+    }
+}
+
+async function putCorpus(data){
+    log.out("DbAccess.putCorpus: Uploading CORPUS data to database")
+    await client.connect();
+    try {
+        var coll = db.collection("corpus");
+        await coll.insertMany(data);
+    } catch (error) {
+        log.out("DbAccess.putCorpus: Error uploading Corpus data to database")
+        log.out(error)
+    }
+};
+
+async function putStations(data){
+    log.out("DbAccess.putStations: Uploading Stations data to database")
+    await client.connect();
+    try {
+        var coll = db.collection("stations");
+        coll.insertMany(data);
+    } catch (error) {
+        log.out("DbAccess.putStations: Error uploading Stations data to database")
+        log.out(error)
+    }
+};
+
+async function updateMeta(type, target, unixTime){
+    await client.connect();
+    var coll = db.collection("meta");
+    var filter = {type: type, target: target};
+    var update = {$set:{updated: unixTime}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await coll.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
+        log.out(err)
+    }
+}
+
+async function query(collection, query){
+    await client.connect();
+    log.out(`dbAccess.query: Connecting to collection: '${collection}'`)
+    var qcoll = db.collection(collection);
+    var qcursor = qcoll.find(query)
+    qcursor.project({_id: 0})
+    log.out(`dbAccess.query: Running Query: ${JSON.stringify(query)}`)
+    increment(collection)
+    return (await qcursor.toArray());
+}
+
+async function ensureIndex(col, field, text) {
+    await client.connect();
+    if (!text) {
+        log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`)
+        db.createIndex(col, field);
+    } else {
+        log.out(`dbAccess.ensureIndex: Creating text index in collection ${col} for field ${field}`)
+        let idx = {}
+        idx[field] = "text";
+        db.createIndex(col, idx);
+    }
+    log.out(`dbAccess.ensureIndex: Index created`);
+    return;
+}
+
+async function increment(target) {
+    await client.connect();
+    let col = db.collection("meta");
+    let update = {}
+    update[target] = 1
+    col.updateOne({target: "counters"}, {$inc:update})
+    return;
+}
+
+async function createCount() {
+    await client.connect();
+    let col = db.collection("meta");
+    var filter = {type: "count", target: "counters"};
+    var update = {$set:{/*since: new Date,*/ type: "count", target: "counters"}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await col.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: count meta added updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to add count`)
+        log.out(err)
+    }
+}
+
+module.exports = {
+    putCorpus,
+    putStations,
+    dropCollection,
+    updateMeta,
+    query,
+    ensureIndex,
+    increment,
+    createCount
+}

src/services/find.services.js

@@ -0,0 +1,59 @@
+// Parse and return a find request
+
+const log = require('../utils/log.utils');          // Log Helper
+const db  = require('../services/dbAccess.services');
+const san = require('../utils/sanitizer.utils')
+
+// DB Query: query(collection, query)
+
+// Define collection as all queries are for the "corpus" collection.
+const col = "corpus"
+
+async function name(id){
+    log.out(`findServices.name: Finding station name: ${id}`)
+    var name = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {NLCDESC: name}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function crs(id){
+    log.out(`findServices.crs: Finding crs: ${id}`)
+    var crs = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {'3ALPHA': crs}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function nlc(id){
+    log.out(`findServices.nlc: Finding nlc: ${id}`)
+    var nlc = san.cleanApiEndpointNum(id)
+    query = {NLC: parseInt(nlc)}
+    log.out(`findServices.nlc: NLC Converted to int: ${query}`)
+    var data = await db.query(col,query)
+    return data
+}
+
+async function tiploc(id){
+    log.out(`findServices.tiploc: Finding tiploc: ${id}`)
+    var tiploc = san.cleanApiEndpointTxt(id.toUpperCase())
+    query = {TIPLOC: tiploc}
+    var data = await db.query(col,query)
+    return data
+}
+
+async function stanox(id){
+    log.out(`findServices.stanox: Finding stanox: ${id}`)
+    var stanox = san.cleanApiEndpointNum(id)
+    query = {STANOX: String(stanox)}
+    var data = await db.query(col,query)
+    return data
+}
+
+module.exports = {
+    name,
+    crs,
+    nlc,
+    tiploc,
+    stanox
+}

src/services/issue.services.js

@@ -0,0 +1,33 @@
+const axios  = require('axios')
+const log    = require('../utils/log.utils')
+
+async function processor(data) {
+    log.out(`issueService.processor: Issue received`)
+    let out = {}
+    out.title = data.subject.replace(/<[^>]+>|[\*\$]/g, '');
+    out.body = data.msg.replace(/<[^>]+>|[\*\$]/g, '')
+    sendToGitea(out);
+}
+
+async function sendToGitea(body) {
+    let key = process.env.OWL_GIT_ISSUEBOT
+    let url = process.env.OWL_GIT_APIENDPOINT
+    let opts = {
+        headers: {
+            Authorization: key
+        }
+    }
+    var res = await axios.post(url, body, opts)
+    // Need to read the output from the POST and pass the result upwards to the client.
+    if (res.status == 201) {
+        log.out("issueService.sendToGitea: Issue sent to Gitea")
+        return {status: res.status,message:"issue created"}
+    } else {
+        log.out("issueService.sendToGitea: Failed to send issue to Gitea")
+        return {status: res.status,message:"issue not created"}
+    }
+}
+
+module.exports = {
+    processor
+}

src/services/kube.services.js

@@ -0,0 +1,20 @@
+async function getAlive(){
+    log.out(`kubeServices.getAlive: alive hook checked`)
+    return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
+}
+
+async function getReady(){
+    log.out(`kubeServices.getReady: ready hook checked`)
+    return "not_implemented";
+};
+
+async function getTime(){
+    var now = new Date()
+    return {responseGenerated: now}
+}
+
+module.exports = {
+    getAlive,
+    getReady,
+    getTime
+}

src/services/ldb.services.js

@@ -0,0 +1,52 @@
+// Parse and return an LDB Request
+
+// FUNCTIONS
+//  post(body, id): Exported:
+            // body: [req.body from controller]
+            // id  : [req.params.id from controller - this is expected to be CRS or TIPLOC]
+
+//  convertTiploc(TIPLOC)  : Exported: Looks up CRS, Name & STANOX for Tiploc
+
+const log  = require('../utils/log.utils');          // Log Helper
+const ldb  = require('ldbs-json')
+const util = require('../utils/ldb.utils')
+const san  = require('../utils/sanitizer.utils')
+const db   = require('../services/dbAccess.services')
+
+const ldbKey   = process.env.OWL_LDB_KEY
+const ldbsvKey = process.env.OWL_LDB_SVKEY
+
+async function get(body, id){
+    var cleanId = san.cleanApiEndpointTxt(id);
+    var obj = await util.checkCrs(cleanId);
+    try {
+        var crs = obj[0]['3ALPHA'];
+        log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`);
+        var data = await arrDepBoard(crs);
+        db.increment("ldbws") // Need to add creation of this document to the database. >> {type:"count",counting:"api_hit",target:"ldbws",since:"DATE"}
+    } catch (err) {
+        log.out(`ldbService.get: Error, Unable to find CRS: ${err}`)
+        var data = {ERROR:'NOT_FOUND',description:'The entered station was not found.  Please check and try again.'};
+    }
+    return data;
+}
+
+async function arrDepBoard(CRS){
+    log.out(`ldbService.arrDepBoard: Trying to fetch ArrDep Board for ${CRS}`)
+    try {
+        var options = {
+            numRows: 10,
+            crs: CRS.toUpperCase()
+        }
+        var api = new ldb(ldbKey,false)
+        var reply = await api.call("GetArrDepBoardWithDetails",options)
+        return reply
+    } catch (err) {
+        log.out(`ldbService.arrDepBoard: Lookup Failed for: ${CRS}`)
+        return {GetStationBoardResult: "not available", Reason: `The CRS code ${CRS} is not valid`, Why: `Sometimes a station will have more than one CRS - for example Filton Abbey Wood has FIT and FAW however schedules are only available when looking up with FIT - this is how the National Rail Enquiries systems work.`};
+    }
+};
+
+module.exports = {
+    get
+}

src/services/list.services.js

@@ -0,0 +1,20 @@
+const log    = require('../utils/log.utils');          // Log Helper
+const db     = require('../services/dbAccess.services')
+const os     = require('os')
+
+async function getStations(){
+    var out = await db.query("stations")
+    log.out(`listServices.getStations: fetched stations list`)
+    return out;
+}
+
+async function getCorpus(){
+    var out = await db.query("corpus")
+    log.out(`listServices.getCorpus: fetched CORPUS list`)
+    return out;
+}
+
+module.exports = {
+    getStations,
+    getCorpus
+}

src/services/stats.services.js

@@ -0,0 +1,16 @@
+const log    = require('../utils/log.utils');          // Log Helper
+const db     = require('../services/dbAccess.services')
+const os     = require('os')
+
+async function hits(){
+    var dat = await db.query("meta", {target: "counters"});
+    log.out(`listServices.meta: fetched server meta`)
+    let out = {}
+    out.host = os.hostname()
+    out.dat = dat
+    return out;
+}
+
+module.exports = {
+    hits
+}