dbInit now working in any database state

UpNext.md

@@ -1,5 +1,3 @@
 # What to do next:
 
-* Cannot drop a MongoDB Collection if it doesn't exist.  Need to wrap a the drop within an if block.  - dbAccess.services
+* Ensure existing 'meta' entries are updated and new entries aren't made.
-
-* Need to check age of Corpus data and only update if not present or older than 14 days (1209600000ms)

app.js

@@ -21,7 +21,7 @@ const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0"
 const srvPort   = process.env.OWL_SRV_PORT || 8460
 
 // Print version number:
-log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
+log.out(`app: Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
 
 // Test for required vars:
     // const varTest = require('./src/utils/varTest.utils');
@@ -55,9 +55,9 @@ app.use('/api/v1/kube', kubeRtr);
 // Start Express
 app.listen(srvPort, srvListen, (error) =>{
   if(!error) {
-      log.out(`Server: Listening on http://${srvListen}:${srvPort}`);
+      log.out(`app.listen: Listening on http://${srvListen}:${srvPort}`);
-      log.out("State: alive")
+      log.out("app.listen: State - alive")
   } else {
-      log.out("Error occurred, server can't start", error);
+      log.out("app.listen: Error occurred, server can't start", error);
   }
 });

src/services/corpus.services.js

@@ -29,7 +29,7 @@ async function get(){
 }
 
 async function fetch() {
-    log.out("Corpus: Fetching CORPUS Data from Network Rail")
+    log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
     authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
     const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
     const options = {
@@ -40,16 +40,16 @@ async function fetch() {
     }
     try {
         var { data } = await axios.get(url, options)
-        log.out("Corpus: CORPUS Data fetched")
+        log.out("corpus.fetch: CORPUS Data fetched")
     } catch (error) {
-        log.out("Corpus: Error fetching CORPUS")
+        log.out("corpus.fetch: Error fetching CORPUS")
         log.out(error)
     }
     return data
 }
 
 async function extract(input) {
-    log.out(`Corpus: Extracting CORPUS archive`)
+    log.out(`corpus.extract: Extracting CORPUS archive`)
     var raw = await gz.ungzip(input)
     var obj = await JSON.parse(raw)
     var output = obj.TIPLOCDATA
@@ -57,7 +57,7 @@ async function extract(input) {
 }
 
 async function clean(input) {
-    log.out(`Corpus: Removing non-stations from CORPUS data`)
+    log.out(`corpus.clean: Removing non-stations from CORPUS data`)
     let clean = [];
     for (const element of input) {
         if (element.STANOX != ' ' && element['3ALPHA'] != ' '){

src/services/dbAccess.services.js

@@ -13,20 +13,24 @@ const client = new MongoClient(uri);
 const db = client.db(dbName);
 
 async function dropCollection(coll){
-    log.out(`DbAccess.dropCollection: checking if collection exists: ${coll}`)
-
-    //Some Code Here:
     await client.connect();
-    var cols = await db.listCollections().toArray()
-    log.out(`dbAccess.dropCollection: Existing collections: ${JSON.stringify(cols)}`)
-    // If (any object in the Array contains name:{coll} then exsists.  If not doesn't exist.)
 
-
+    // check if collection contains any documents, if it doesn't, it is either empty or non-existent - it doesn't need dropping.
-    // If Collection Exists:
+    var collection = db.collection(coll);
-    log.out(`DbAccess.dropCollection: dropping collection: ${coll}`)
+    var count = await collection.countDocuments();
+    log.out(`DbAccess.dropCollection: Collection '${coll}' contains ${count} documents`)
+    if (count == 0) {
+        log.out(`DbAccess.dropCollection: Collection '${coll}' is empty.  Do not need to drop`)
+    } else {
+        log.out(`DbAccess.dropCollection: dropping collection: '${coll}'`)
         db.dropCollection(coll);
-    log.out(`DbAccess.dropCollection: dropped collection: ${coll}`)
+        log.out(`DbAccess.dropCollection: dropped collection: '${coll}'`)
-    // Else Do Nothing
+    }
+    // ALTERNATIVE METHOD - Appears to not be needed:
+    // This code returns and prints an array containing an object for each existing collection.
+//    var cols = await db.listCollections().toArray()
+//    log.out(`dbAccess.dropCollection: Existing collections: ${JSON.stringify(cols)}`)
+    // If (any object in the Array contains name:{coll} then exsists.  If not doesn't exist.)
 }
 
 async function putCorpus(data){
@@ -53,10 +57,20 @@ async function putStations(data){
     }
 };
 
-async function putMeta(data){
+async function updateMeta(type, target, unixTime){
     await client.connect();
     var coll = db.collection("meta");
-    coll.insertMany(data);
+    var filter = {type: type, target: target};
+    var update = {$set:{updated: unixTime}};
+    var options = {upsert: true};  // If document isn't present will insert.
+    try {
+        var result = await coll.updateOne(filter,update,options)
+        log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
+        log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
+    } catch (err) {
+        log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
+        log.out(err)
+    }
 }
 
 async function query(collection, query){
@@ -71,6 +85,6 @@ module.exports = {
     putCorpus,
     putStations,
     dropCollection,
-    putMeta,
+    updateMeta,
     query
 }

src/services/kube.services.js

@@ -1,8 +1,10 @@
 async function getAlive(){
+    log.out(`kubeServices.getAlive: alive hook checked`)
     return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
 }
 
 async function getReady(){
+    log.out(`kubeServices.getReady: ready hook checked`)
     return "not_implemented";
 };
 

src/services/ldb.services.js

@@ -29,7 +29,7 @@ async function get(body, id){
 
 async function arrDepBoard(CRS){
     var valid = await util.checkCrs(CRS)
-    log.out(`ldbService: Fetching ArrDep Board for ${CRS}`)
+    log.out(`ldbService.arrDepBoard: Fetching ArrDep Board for ${CRS}`)
     if (valid != false){
         var options = {
             numRows: 10,
@@ -39,6 +39,7 @@ async function arrDepBoard(CRS){
         var reply = await api.call("GetArrDepBoardWithDetails",options)
         return reply
     } else if (valid == false) {
+        log.out(`ldbService.arrDepBoard: Invalid 3ALPHA for lookup: ${CRS}`)
         return {status: false, reason: "Invalid CRS/3alpha code"};
     }
 };

src/utils/dbinit.utils.js

@@ -1,57 +1,79 @@
 // FUNCTIONS
 //  init()   : Exported: Uses the internal functions to initialise databases.
-//  check()  : 
+//  check()  : Checks data presence and age.
-//  build()  : Adds CORPUS Data to DB.
+//  build()  : Builds/Rebuilds collections.
 
 const log      = require('../utils/log.utils');          // Log Helper
+const time     = require('../utils/timeConvert.utils');  // Time Helper
 const corpus   = require('../services/corpus.services');
 const dbAccess = require('../services/dbAccess.services');
 
 async function init(){
-    var status = await check();
+    var status = await check('corpus');
     if (status == "not_ready") {
         try {
-            build("all")
+            build("corpus")
         } catch (err) {
-            log.out("dbInit.init: Error building database")
+            log.out("dbInitUtils.init: Error building corpus database")
+            log.out(err)
+        }
+    }
+
+    var status = await check('stations')
+    if (status == "not_ready") {
+        try {
+            build("stations")
+        } catch (err) {
+            log.out("dbInitUtils.init: Error building stations database")
             log.out(err)
         }
     }
 }
 
-async function check(){
+async function check(coll){
-    log.out("DbInit.check: Checking database state")
+    log.out(`dbInitUtils.check: Checking collection '${coll}'`)
-    let test = {'collection': 'any'};
     try {
-        await dbAccess.query('meta',test);
+        var queryStr = {'type':'collection','target': coll};
-        log.out(`DbInit.check: Reading Database Collection: meta`);
+        var res = await dbAccess.query('meta',queryStr);
-        return "not_ready"
+        log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
-    } catch (error) {
+        var now = time.jsUnix(Date.now())
-        log.out(error)
+        var delta = now - res['0']['updated']
+    } catch (err) {
+        log.out(`dbInitUtils.check: Unable to find out data age.  Presume stale. Error Message:`)
+        log.out(err)
+        var delta = 12096000 // Extra zero to ensure data is updated.
+    }
+
+    var maxAge = 1209600 // 14 Days
+    if (delta > maxAge) {
+        log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds.  Update pending`)
         return "not_ready"
+    } else {
+        log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds.  Update not required`)
+        return "ready"
     }
 }
 
 async function build(db){  // `db` must be one of: `corpus`, `stations`, `all`.
-    log.out("DbInit.build: Building database structure")
+    log.out("dbInitUtils.build: Building database structure")
     var corpusAll = await corpus.get();
-    if (db === "corpus" || "all") {
+    if (db === "corpus") {
         await dbAccess.dropCollection("corpus");
         dbAccess.putCorpus(corpusAll);
-        meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
+
-        dbAccess.putMeta(meta);
+        log.out(`dbInitUtils.build: Updateing corpus meta`)
+        dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
     }
-    if (db === "stations" || "all") {
+    if (db === "stations") {
         await dbAccess.dropCollection("stations");
         var corpusSubset = await corpus.subset(corpusAll);
         dbAccess.putStations(corpusSubset);
-        meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
+
-        dbAccess.putMeta(meta);
+        log.out(`dbInitUtils.build: Updating stations meta`)
+        dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
     }
 }
 
 module.exports = {
-    init,
+    init
-    check,
-    build
 }

src/utils/timeConvert.utils.js

@@ -0,0 +1,15 @@
+function unixLocal(unix) {
+    var jsTime = unix*1000
+    var dt = new Date(jsTime)
+    return dt.toLocaleString()
+}
+
+function jsUnix(js) {
+    var preRound = js / 1000
+    return Math.round(preRound)
+}
+
+module.exports = {
+    unixLocal,
+    jsUnix,
+}