UpNext.md

@@ -1,3 +1,5 @@
 # What to do next:
 
-* Ensure existing 'meta' entries are updated and new entries aren't made.
+* Cannot drop a MongoDB Collection if it doesn't exist.  Need to wrap a the drop within an if block.  - dbAccess.services
+
+* Need to check age of Corpus data and only update if not present or older than 14 days (1209600000ms)

app.js

@@ -21,7 +21,7 @@ const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0"
 const srvPort   = process.env.OWL_SRV_PORT || 8460
 
 // Print version number:
-log.out(`app: Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
+log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
 
 // Test for required vars:
     // const varTest = require('./src/utils/varTest.utils');
@@ -55,9 +55,9 @@ app.use('/api/v1/kube', kubeRtr);
 // Start Express
 app.listen(srvPort, srvListen, (error) =>{
   if(!error) {
-      log.out(`app.listen: Listening on http://${srvListen}:${srvPort}`);
-      log.out("app.listen: State - alive")
+      log.out(`Server: Listening on http://${srvListen}:${srvPort}`);
+      log.out("State: alive")
   } else {
-      log.out("app.listen: Error occurred, server can't start", error);
+      log.out("Error occurred, server can't start", error);
   }
 });

src/services/corpus.services.js

@@ -29,7 +29,7 @@ async function get(){
 }
 
 async function fetch() {
-    log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
+    log.out("Corpus: Fetching CORPUS Data from Network Rail")
     authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
     const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
     const options = {
@@ -40,16 +40,16 @@ async function fetch() {
     }
     try {
         var { data } = await axios.get(url, options)
-        log.out("corpus.fetch: CORPUS Data fetched")
+        log.out("Corpus: CORPUS Data fetched")
     } catch (error) {
-        log.out("corpus.fetch: Error fetching CORPUS")
+        log.out("Corpus: Error fetching CORPUS")
         log.out(error)
     }
     return data
 }
 
 async function extract(input) {
-    log.out(`corpus.extract: Extracting CORPUS archive`)
+    log.out(`Corpus: Extracting CORPUS archive`)
     var raw = await gz.ungzip(input)
     var obj = await JSON.parse(raw)
     var output = obj.TIPLOCDATA
@@ -57,7 +57,7 @@ async function extract(input) {
 }
 
 async function clean(input) {
-    log.out(`corpus.clean: Removing non-stations from CORPUS data`)
+    log.out(`Corpus: Removing non-stations from CORPUS data`)
     let clean = [];
     for (const element of input) {
         if (element.STANOX != ' ' && element['3ALPHA'] != ' '){

src/services/dbAccess.services.js

@@ -13,24 +13,20 @@ const client = new MongoClient(uri);
 const db = client.db(dbName);
 
 async function dropCollection(coll){
-    await client.connect();
+    log.out(`DbAccess.dropCollection: checking if collection exists: ${coll}`)
 
-    // check if collection contains any documents, if it doesn't, it is either empty or non-existent - it doesn't need dropping.
-    var collection = db.collection(coll);
-    var count = await collection.countDocuments();
-    log.out(`DbAccess.dropCollection: Collection '${coll}' contains ${count} documents`)
-    if (count == 0) {
-        log.out(`DbAccess.dropCollection: Collection '${coll}' is empty.  Do not need to drop`)
-    } else {
-        log.out(`DbAccess.dropCollection: dropping collection: '${coll}'`)
-        db.dropCollection(coll);
-        log.out(`DbAccess.dropCollection: dropped collection: '${coll}'`)
-    }
-    // ALTERNATIVE METHOD - Appears to not be needed:
-    // This code returns and prints an array containing an object for each existing collection.
-//    var cols = await db.listCollections().toArray()
-//    log.out(`dbAccess.dropCollection: Existing collections: ${JSON.stringify(cols)}`)
+    //Some Code Here:
+    await client.connect();
+    var cols = await db.listCollections().toArray()
+    log.out(`dbAccess.dropCollection: Existing collections: ${JSON.stringify(cols)}`)
     // If (any object in the Array contains name:{coll} then exsists.  If not doesn't exist.)
+
+
+    // If Collection Exists:
+    log.out(`DbAccess.dropCollection: dropping collection: ${coll}`)
+    db.dropCollection(coll);
+    log.out(`DbAccess.dropCollection: dropped collection: ${coll}`)
+    // Else Do Nothing
 }
 
 async function putCorpus(data){
@@ -57,20 +53,10 @@ async function putStations(data){
     }
 };
 
-async function updateMeta(type, target, unixTime){
+async function putMeta(data){
     await client.connect();
     var coll = db.collection("meta");
-    var filter = {type: type, target: target};
-    var update = {$set:{updated: unixTime}};
-    var options = {upsert: true};  // If document isn't present will insert.
-    try {
-        var result = await coll.updateOne(filter,update,options)
-        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
-        log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
-    } catch (err) {
-        log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
-        log.out(err)
-    }
+    coll.insertMany(data);
 }
 
 async function query(collection, query){
@@ -85,6 +71,6 @@ module.exports = {
     putCorpus,
     putStations,
     dropCollection,
-    updateMeta,
+    putMeta,
     query
 }

src/services/kube.services.js

@@ -1,10 +1,8 @@
 async function getAlive(){
-    log.out(`kubeServices.getAlive: alive hook checked`)
     return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
 }
 
 async function getReady(){
-    log.out(`kubeServices.getReady: ready hook checked`)
     return "not_implemented";
 };
 

src/services/ldb.services.js

@@ -29,7 +29,7 @@ async function get(body, id){
 
 async function arrDepBoard(CRS){
     var valid = await util.checkCrs(CRS)
-    log.out(`ldbService.arrDepBoard: Fetching ArrDep Board for ${CRS}`)
+    log.out(`ldbService: Fetching ArrDep Board for ${CRS}`)
     if (valid != false){
         var options = {
             numRows: 10,
@@ -39,7 +39,6 @@ async function arrDepBoard(CRS){
         var reply = await api.call("GetArrDepBoardWithDetails",options)
         return reply
     } else if (valid == false) {
-        log.out(`ldbService.arrDepBoard: Invalid 3ALPHA for lookup: ${CRS}`)
         return {status: false, reason: "Invalid CRS/3alpha code"};
     }
 };

src/utils/dbinit.utils.js

@@ -1,79 +1,57 @@
 // FUNCTIONS
 //  init()   : Exported: Uses the internal functions to initialise databases.
-//  check()  : Checks data presence and age.
-//  build()  : Builds/Rebuilds collections.
+//  check()  : 
+//  build()  : Adds CORPUS Data to DB.
 
 const log      = require('../utils/log.utils');          // Log Helper
-const time     = require('../utils/timeConvert.utils');  // Time Helper
 const corpus   = require('../services/corpus.services');
 const dbAccess = require('../services/dbAccess.services');
 
 async function init(){
-    var status = await check('corpus');
+    var status = await check();
     if (status == "not_ready") {
         try {
-            build("corpus")
+            build("all")
         } catch (err) {
-            log.out("dbInitUtils.init: Error building corpus database")
-            log.out(err)
-        }
-    }
-
-    var status = await check('stations')
-    if (status == "not_ready") {
-        try {
-            build("stations")
-        } catch (err) {
-            log.out("dbInitUtils.init: Error building stations database")
+            log.out("dbInit.init: Error building database")
             log.out(err)
         }
     }
 }
 
-async function check(coll){
-    log.out(`dbInitUtils.check: Checking collection '${coll}'`)
+async function check(){
+    log.out("DbInit.check: Checking database state")
+    let test = {'collection': 'any'};
     try {
-        var queryStr = {'type':'collection','target': coll};
-        var res = await dbAccess.query('meta',queryStr);
-        log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
-        var now = time.jsUnix(Date.now())
-        var delta = now - res['0']['updated']
-    } catch (err) {
-        log.out(`dbInitUtils.check: Unable to find out data age.  Presume stale. Error Message:`)
-        log.out(err)
-        var delta = 12096000 // Extra zero to ensure data is updated.
-    }
-
-    var maxAge = 1209600 // 14 Days
-    if (delta > maxAge) {
-        log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds.  Update pending`)
+        await dbAccess.query('meta',test);
+        log.out(`DbInit.check: Reading Database Collection: meta`);
+        return "not_ready"
+    } catch (error) {
+        log.out(error)
         return "not_ready"
-    } else {
-        log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds.  Update not required`)
-        return "ready"
     }
 }
 
 async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
-    log.out("dbInitUtils.build: Building database structure")
+    log.out("DbInit.build: Building database structure")
     var corpusAll = await corpus.get();
-    if (db === "corpus") {
+    if (db === "corpus" || "all") {
         await dbAccess.dropCollection("corpus");
         dbAccess.putCorpus(corpusAll);
-
-        log.out(`dbInitUtils.build: Updateing corpus meta`)
-        dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
+        meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
+        dbAccess.putMeta(meta);
     }
-    if (db === "stations") {
+    if (db === "stations" || "all") {
         await dbAccess.dropCollection("stations");
         var corpusSubset = await corpus.subset(corpusAll);
         dbAccess.putStations(corpusSubset);
-
-        log.out(`dbInitUtils.build: Updating stations meta`)
-        dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
+        meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
+        dbAccess.putMeta(meta);
     }
 }
 
 module.exports = {
-    init
+    init,
+    check,
+    build
 }

src/utils/timeConvert.utils.js

@@ -1,15 +0,0 @@
-function unixLocal(unix) {
-    var jsTime = unix*1000
-    var dt = new Date(jsTime)
-    return dt.toLocaleString()
-}
-
-function jsUnix(js) {
-    var preRound = js / 1000
-    return Math.round(preRound)
-}
-
-module.exports = {
-    unixLocal,
-    jsUnix,
-}