dbInit now working in any database state

This commit is contained in:
Fred Boniface 2023-01-11 13:12:10 +00:00
parent 0317dc723a
commit bc23a976f9
8 changed files with 87 additions and 39 deletions

View File

@ -1,5 +1,3 @@
# What to do next: # What to do next:
* Cannot drop a MongoDB Collection if it doesn't exist. Need to wrap a the drop within an if block. - dbAccess.services * Ensure existing 'meta' entries are updated and new entries aren't made.
* Need to check age of Corpus data and only update if not present or older than 14 days (1209600000ms)

8
app.js
View File

@ -21,7 +21,7 @@ const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0"
const srvPort = process.env.OWL_SRV_PORT || 8460 const srvPort = process.env.OWL_SRV_PORT || 8460
// Print version number: // Print version number:
log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); log.out(`app: Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
// Test for required vars: // Test for required vars:
// const varTest = require('./src/utils/varTest.utils'); // const varTest = require('./src/utils/varTest.utils');
@ -55,9 +55,9 @@ app.use('/api/v1/kube', kubeRtr);
// Start Express // Start Express
app.listen(srvPort, srvListen, (error) =>{ app.listen(srvPort, srvListen, (error) =>{
if(!error) { if(!error) {
log.out(`Server: Listening on http://${srvListen}:${srvPort}`); log.out(`app.listen: Listening on http://${srvListen}:${srvPort}`);
log.out("State: alive") log.out("app.listen: State - alive")
} else { } else {
log.out("Error occurred, server can't start", error); log.out("app.listen: Error occurred, server can't start", error);
} }
}); });

View File

@ -29,7 +29,7 @@ async function get(){
} }
async function fetch() { async function fetch() {
log.out("Corpus: Fetching CORPUS Data from Network Rail") log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64') authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS' const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
const options = { const options = {
@ -40,16 +40,16 @@ async function fetch() {
} }
try { try {
var { data } = await axios.get(url, options) var { data } = await axios.get(url, options)
log.out("Corpus: CORPUS Data fetched") log.out("corpus.fetch: CORPUS Data fetched")
} catch (error) { } catch (error) {
log.out("Corpus: Error fetching CORPUS") log.out("corpus.fetch: Error fetching CORPUS")
log.out(error) log.out(error)
} }
return data return data
} }
async function extract(input) { async function extract(input) {
log.out(`Corpus: Extracting CORPUS archive`) log.out(`corpus.extract: Extracting CORPUS archive`)
var raw = await gz.ungzip(input) var raw = await gz.ungzip(input)
var obj = await JSON.parse(raw) var obj = await JSON.parse(raw)
var output = obj.TIPLOCDATA var output = obj.TIPLOCDATA
@ -57,7 +57,7 @@ async function extract(input) {
} }
async function clean(input) { async function clean(input) {
log.out(`Corpus: Removing non-stations from CORPUS data`) log.out(`corpus.clean: Removing non-stations from CORPUS data`)
let clean = []; let clean = [];
for (const element of input) { for (const element of input) {
if (element.STANOX != ' ' && element['3ALPHA'] != ' '){ if (element.STANOX != ' ' && element['3ALPHA'] != ' '){

View File

@ -57,10 +57,20 @@ async function putStations(data){
} }
}; };
async function putMeta(data){ async function updateMeta(type, target, unixTime){
await client.connect(); await client.connect();
var coll = db.collection("meta"); var coll = db.collection("meta");
coll.insertMany(data); var filter = {type: type, target: target};
var update = {$set:{updated: unixTime}};
var options = {upsert: true}; // If document isn't present will insert.
try {
var result = await coll.updateOne(filter,update,options)
log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
} catch (err) {
log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
log.out(err)
}
} }
async function query(collection, query){ async function query(collection, query){
@ -75,6 +85,6 @@ module.exports = {
putCorpus, putCorpus,
putStations, putStations,
dropCollection, dropCollection,
putMeta, updateMeta,
query query
} }

View File

@ -1,8 +1,10 @@
async function getAlive(){ async function getAlive(){
log.out(`kubeServices.getAlive: alive hook checked`)
return {code: 200, state: {state: "alive",noise: "twit-twoo"}} return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
} }
async function getReady(){ async function getReady(){
log.out(`kubeServices.getReady: ready hook checked`)
return "not_implemented"; return "not_implemented";
}; };

View File

@ -29,7 +29,7 @@ async function get(body, id){
async function arrDepBoard(CRS){ async function arrDepBoard(CRS){
var valid = await util.checkCrs(CRS) var valid = await util.checkCrs(CRS)
log.out(`ldbService: Fetching ArrDep Board for ${CRS}`) log.out(`ldbService.arrDepBoard: Fetching ArrDep Board for ${CRS}`)
if (valid != false){ if (valid != false){
var options = { var options = {
numRows: 10, numRows: 10,
@ -39,6 +39,7 @@ async function arrDepBoard(CRS){
var reply = await api.call("GetArrDepBoardWithDetails",options) var reply = await api.call("GetArrDepBoardWithDetails",options)
return reply return reply
} else if (valid == false) { } else if (valid == false) {
log.out(`ldbService.arrDepBoard: Invalid 3ALPHA for lookup: ${CRS}`)
return {status: false, reason: "Invalid CRS/3alpha code"}; return {status: false, reason: "Invalid CRS/3alpha code"};
} }
}; };

View File

@ -1,57 +1,79 @@
// FUNCTIONS // FUNCTIONS
// init() : Exported: Uses the internal functions to initialise databases. // init() : Exported: Uses the internal functions to initialise databases.
// check() : // check() : Checks data presence and age.
// build() : Adds CORPUS Data to DB. // build() : Builds/Rebuilds collections.
const log = require('../utils/log.utils'); // Log Helper const log = require('../utils/log.utils'); // Log Helper
const time = require('../utils/timeConvert.utils'); // Time Helper
const corpus = require('../services/corpus.services'); const corpus = require('../services/corpus.services');
const dbAccess = require('../services/dbAccess.services'); const dbAccess = require('../services/dbAccess.services');
async function init(){ async function init(){
var status = await check(); var status = await check('corpus');
if (status == "not_ready") { if (status == "not_ready") {
try { try {
build("all") build("corpus")
} catch (err) { } catch (err) {
log.out("dbInit.init: Error building database") log.out("dbInitUtils.init: Error building corpus database")
log.out(err)
}
}
var status = await check('stations')
if (status == "not_ready") {
try {
build("stations")
} catch (err) {
log.out("dbInitUtils.init: Error building stations database")
log.out(err) log.out(err)
} }
} }
} }
async function check(){ async function check(coll){
log.out("DbInit.check: Checking database state") log.out(`dbInitUtils.check: Checking collection '${coll}'`)
let test = {'collection': 'any'};
try { try {
await dbAccess.query('meta',test); var queryStr = {'type':'collection','target': coll};
log.out(`DbInit.check: Reading Database Collection: meta`); var res = await dbAccess.query('meta',queryStr);
return "not_ready" log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
} catch (error) { var now = time.jsUnix(Date.now())
log.out(error) var delta = now - res['0']['updated']
} catch (err) {
log.out(`dbInitUtils.check: Unable to find out data age. Presume stale. Error Message:`)
log.out(err)
var delta = 12096000 // Extra zero to ensure data is updated.
}
var maxAge = 1209600 // 14 Days
if (delta > maxAge) {
log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds. Update pending`)
return "not_ready" return "not_ready"
} else {
log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds. Update not required`)
return "ready"
} }
} }
async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`. async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`.
log.out("DbInit.build: Building database structure") log.out("dbInitUtils.build: Building database structure")
var corpusAll = await corpus.get(); var corpusAll = await corpus.get();
if (db === "corpus" || "all") { if (db === "corpus") {
await dbAccess.dropCollection("corpus"); await dbAccess.dropCollection("corpus");
dbAccess.putCorpus(corpusAll); dbAccess.putCorpus(corpusAll);
meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
dbAccess.putMeta(meta); log.out(`dbInitUtils.build: Updateing corpus meta`)
dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
} }
if (db === "stations" || "all") { if (db === "stations") {
await dbAccess.dropCollection("stations"); await dbAccess.dropCollection("stations");
var corpusSubset = await corpus.subset(corpusAll); var corpusSubset = await corpus.subset(corpusAll);
dbAccess.putStations(corpusSubset); dbAccess.putStations(corpusSubset);
meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
dbAccess.putMeta(meta); log.out(`dbInitUtils.build: Updating stations meta`)
dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
} }
} }
module.exports = { module.exports = {
init, init
check,
build
} }

View File

@ -0,0 +1,15 @@
function unixLocal(unix) {
var jsTime = unix*1000
var dt = new Date(jsTime)
return dt.toLocaleString()
}
function jsUnix(js) {
var preRound = js / 1000
return Math.round(preRound)
}
module.exports = {
unixLocal,
jsUnix,
}