From bc23a976f93ce27c086d88ec4e7d82bf30be2e23 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Wed, 11 Jan 2023 13:12:10 +0000 Subject: [PATCH] dbInit now working in any database state --- UpNext.md | 4 +- app.js | 8 ++-- src/services/corpus.services.js | 10 ++--- src/services/dbAccess.services.js | 16 ++++++-- src/services/kube.services.js | 2 + src/services/ldb.services.js | 3 +- src/utils/dbinit.utils.js | 68 ++++++++++++++++++++----------- src/utils/timeConvert.utils.js | 15 +++++++ 8 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 src/utils/timeConvert.utils.js diff --git a/UpNext.md b/UpNext.md index 1b541e5..348163a 100644 --- a/UpNext.md +++ b/UpNext.md @@ -1,5 +1,3 @@ # What to do next: -* Cannot drop a MongoDB Collection if it doesn't exist. Need to wrap a the drop within an if block. - dbAccess.services - -* Need to check age of Corpus data and only update if not present or older than 14 days (1209600000ms) \ No newline at end of file +* Ensure existing 'meta' entries are updated and new entries aren't made. \ No newline at end of file diff --git a/app.js b/app.js index 35b411d..ba509cc 100644 --- a/app.js +++ b/app.js @@ -21,7 +21,7 @@ const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0" const srvPort = process.env.OWL_SRV_PORT || 8460 // Print version number: -log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); +log.out(`app: Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); // Test for required vars: // const varTest = require('./src/utils/varTest.utils'); @@ -55,9 +55,9 @@ app.use('/api/v1/kube', kubeRtr); // Start Express app.listen(srvPort, srvListen, (error) =>{ if(!error) { - log.out(`Server: Listening on http://${srvListen}:${srvPort}`); - log.out("State: alive") + log.out(`app.listen: Listening on http://${srvListen}:${srvPort}`); + log.out("app.listen: State - alive") } else { - log.out("Error occurred, server can't start", error); + log.out("app.listen: Error occurred, server can't start", error); } }); \ No newline at end of file diff --git a/src/services/corpus.services.js b/src/services/corpus.services.js index 71cbd54..2b9f981 100644 --- a/src/services/corpus.services.js +++ b/src/services/corpus.services.js @@ -29,7 +29,7 @@ async function get(){ } async function fetch() { - log.out("Corpus: Fetching CORPUS Data from Network Rail") + log.out("corpus.fetch: Fetching CORPUS Data from Network Rail") authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64') const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS' const options = { @@ -40,16 +40,16 @@ async function fetch() { } try { var { data } = await axios.get(url, options) - log.out("Corpus: CORPUS Data fetched") + log.out("corpus.fetch: CORPUS Data fetched") } catch (error) { - log.out("Corpus: Error fetching CORPUS") + log.out("corpus.fetch: Error fetching CORPUS") log.out(error) } return data } async function extract(input) { - log.out(`Corpus: Extracting CORPUS archive`) + log.out(`corpus.extract: Extracting CORPUS archive`) var raw = await gz.ungzip(input) var obj = await JSON.parse(raw) var output = obj.TIPLOCDATA @@ -57,7 +57,7 @@ async function extract(input) { } async function clean(input) { - log.out(`Corpus: Removing non-stations from CORPUS data`) + log.out(`corpus.clean: Removing non-stations from CORPUS data`) let clean = []; for (const element of input) { if (element.STANOX != ' ' && element['3ALPHA'] != ' '){ diff --git a/src/services/dbAccess.services.js b/src/services/dbAccess.services.js index 9efbad4..e57e86b 100644 --- a/src/services/dbAccess.services.js +++ b/src/services/dbAccess.services.js @@ -57,10 +57,20 @@ async function putStations(data){ } }; -async function putMeta(data){ +async function updateMeta(type, target, unixTime){ await client.connect(); var coll = db.collection("meta"); - coll.insertMany(data); + var filter = {type: type, target: target}; + var update = {$set:{updated: unixTime}}; + var options = {upsert: true}; // If document isn't present will insert. + try { + var result = await coll.updateOne(filter,update,options) + log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`) + log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`) + } catch (err) { + log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`) + log.out(err) + } } async function query(collection, query){ @@ -75,6 +85,6 @@ module.exports = { putCorpus, putStations, dropCollection, - putMeta, + updateMeta, query } \ No newline at end of file diff --git a/src/services/kube.services.js b/src/services/kube.services.js index dac735b..16bb61c 100644 --- a/src/services/kube.services.js +++ b/src/services/kube.services.js @@ -1,8 +1,10 @@ async function getAlive(){ + log.out(`kubeServices.getAlive: alive hook checked`) return {code: 200, state: {state: "alive",noise: "twit-twoo"}} } async function getReady(){ + log.out(`kubeServices.getReady: ready hook checked`) return "not_implemented"; }; diff --git a/src/services/ldb.services.js b/src/services/ldb.services.js index 9932959..55c8638 100644 --- a/src/services/ldb.services.js +++ b/src/services/ldb.services.js @@ -29,7 +29,7 @@ async function get(body, id){ async function arrDepBoard(CRS){ var valid = await util.checkCrs(CRS) - log.out(`ldbService: Fetching ArrDep Board for ${CRS}`) + log.out(`ldbService.arrDepBoard: Fetching ArrDep Board for ${CRS}`) if (valid != false){ var options = { numRows: 10, @@ -39,6 +39,7 @@ async function arrDepBoard(CRS){ var reply = await api.call("GetArrDepBoardWithDetails",options) return reply } else if (valid == false) { + log.out(`ldbService.arrDepBoard: Invalid 3ALPHA for lookup: ${CRS}`) return {status: false, reason: "Invalid CRS/3alpha code"}; } }; diff --git a/src/utils/dbinit.utils.js b/src/utils/dbinit.utils.js index 611798a..c637a5f 100644 --- a/src/utils/dbinit.utils.js +++ b/src/utils/dbinit.utils.js @@ -1,57 +1,79 @@ // FUNCTIONS // init() : Exported: Uses the internal functions to initialise databases. -// check() : -// build() : Adds CORPUS Data to DB. +// check() : Checks data presence and age. +// build() : Builds/Rebuilds collections. const log = require('../utils/log.utils'); // Log Helper +const time = require('../utils/timeConvert.utils'); // Time Helper const corpus = require('../services/corpus.services'); const dbAccess = require('../services/dbAccess.services'); async function init(){ - var status = await check(); + var status = await check('corpus'); if (status == "not_ready") { try { - build("all") + build("corpus") } catch (err) { - log.out("dbInit.init: Error building database") + log.out("dbInitUtils.init: Error building corpus database") + log.out(err) + } + } + + var status = await check('stations') + if (status == "not_ready") { + try { + build("stations") + } catch (err) { + log.out("dbInitUtils.init: Error building stations database") log.out(err) } } } -async function check(){ - log.out("DbInit.check: Checking database state") - let test = {'collection': 'any'}; +async function check(coll){ + log.out(`dbInitUtils.check: Checking collection '${coll}'`) try { - await dbAccess.query('meta',test); - log.out(`DbInit.check: Reading Database Collection: meta`); - return "not_ready" - } catch (error) { - log.out(error) + var queryStr = {'type':'collection','target': coll}; + var res = await dbAccess.query('meta',queryStr); + log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`) + var now = time.jsUnix(Date.now()) + var delta = now - res['0']['updated'] + } catch (err) { + log.out(`dbInitUtils.check: Unable to find out data age. Presume stale. Error Message:`) + log.out(err) + var delta = 12096000 // Extra zero to ensure data is updated. + } + + var maxAge = 1209600 // 14 Days + if (delta > maxAge) { + log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds. Update pending`) return "not_ready" + } else { + log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds. Update not required`) + return "ready" } } async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`. - log.out("DbInit.build: Building database structure") + log.out("dbInitUtils.build: Building database structure") var corpusAll = await corpus.get(); - if (db === "corpus" || "all") { + if (db === "corpus") { await dbAccess.dropCollection("corpus"); dbAccess.putCorpus(corpusAll); - meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}]; - dbAccess.putMeta(meta); + + log.out(`dbInitUtils.build: Updateing corpus meta`) + dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now())); } - if (db === "stations" || "all") { + if (db === "stations") { await dbAccess.dropCollection("stations"); var corpusSubset = await corpus.subset(corpusAll); dbAccess.putStations(corpusSubset); - meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}]; - dbAccess.putMeta(meta); + + log.out(`dbInitUtils.build: Updating stations meta`) + dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now())); } } module.exports = { - init, - check, - build + init } \ No newline at end of file diff --git a/src/utils/timeConvert.utils.js b/src/utils/timeConvert.utils.js new file mode 100644 index 0000000..852b60d --- /dev/null +++ b/src/utils/timeConvert.utils.js @@ -0,0 +1,15 @@ +function unixLocal(unix) { + var jsTime = unix*1000 + var dt = new Date(jsTime) + return dt.toLocaleString() +} + +function jsUnix(js) { + var preRound = js / 1000 + return Math.round(preRound) +} + +module.exports = { + unixLocal, + jsUnix, +} \ No newline at end of file