fred.boniface/OwlBoard
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

dbInit now working in any database state

Browse Source
This commit is contained in:
Fred Boniface 2023-01-11 13:12:10 +00:00
parent 0317dc723a
commit bc23a976f9
8 changed files with 87 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
UpNext.md
View File

@ -1,5 +1,3 @@
# What to do next:
* Cannot drop a MongoDB Collection if it doesn't exist. Need to wrap a the drop within an if block. - dbAccess.services
* Need to check age of Corpus data and only update if not present or older than 14 days (1209600000ms)
* Ensure existing 'meta' entries are updated and new entries aren't made.

8
app.js
View File

@ -21,7 +21,7 @@ const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0"
const srvPort = process.env.OWL_SRV_PORT || 8460
// Print version number:
log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
log.out(`app: Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
// Test for required vars:
// const varTest = require('./src/utils/varTest.utils');
@ -55,9 +55,9 @@ app.use('/api/v1/kube', kubeRtr);
// Start Express
app.listen(srvPort, srvListen, (error) =>{
if(!error) {
log.out(`Server: Listening on http://${srvListen}:${srvPort}`);
log.out("State: alive")
log.out(`app.listen: Listening on http://${srvListen}:${srvPort}`);
log.out("app.listen: State - alive")
} else {
log.out("Error occurred, server can't start", error);
log.out("app.listen: Error occurred, server can't start", error);
}
});

10
src/services/corpus.services.js
View File

@ -29,7 +29,7 @@ async function get(){
}
async function fetch() {
log.out("Corpus: Fetching CORPUS Data from Network Rail")
log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
const options = {
@ -40,16 +40,16 @@ async function fetch() {
}
try {
var { data } = await axios.get(url, options)
log.out("Corpus: CORPUS Data fetched")
log.out("corpus.fetch: CORPUS Data fetched")
} catch (error) {
log.out("Corpus: Error fetching CORPUS")
log.out("corpus.fetch: Error fetching CORPUS")
log.out(error)
}
return data
}
async function extract(input) {
log.out(`Corpus: Extracting CORPUS archive`)
log.out(`corpus.extract: Extracting CORPUS archive`)
var raw = await gz.ungzip(input)
var obj = await JSON.parse(raw)
var output = obj.TIPLOCDATA
@ -57,7 +57,7 @@ async function extract(input) {
}
async function clean(input) {
log.out(`Corpus: Removing non-stations from CORPUS data`)
log.out(`corpus.clean: Removing non-stations from CORPUS data`)
let clean = [];
for (const element of input) {
if (element.STANOX != ' ' && element['3ALPHA'] != ' '){

16
src/services/dbAccess.services.js
View File

@ -57,10 +57,20 @@ async function putStations(data){
}
};
async function putMeta(data){
async function updateMeta(type, target, unixTime){
await client.connect();
var coll = db.collection("meta");
coll.insertMany(data);
var filter = {type: type, target: target};
var update = {$set:{updated: unixTime}};
var options = {upsert: true}; // If document isn't present will insert.
try {
var result = await coll.updateOne(filter,update,options)
log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
} catch (err) {
log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
log.out(err)
}
}
async function query(collection, query){
@ -75,6 +85,6 @@ module.exports = {
putCorpus,
putStations,
dropCollection,
putMeta,
updateMeta,
query
}

2
src/services/kube.services.js
View File

@ -1,8 +1,10 @@
async function getAlive(){
log.out(`kubeServices.getAlive: alive hook checked`)
return {code: 200, state: {state: "alive",noise: "twit-twoo"}}
}
async function getReady(){
log.out(`kubeServices.getReady: ready hook checked`)
return "not_implemented";
};

3
src/services/ldb.services.js
View File

@ -29,7 +29,7 @@ async function get(body, id){
async function arrDepBoard(CRS){
var valid = await util.checkCrs(CRS)
log.out(`ldbService: Fetching ArrDep Board for ${CRS}`)
log.out(`ldbService.arrDepBoard: Fetching ArrDep Board for ${CRS}`)
if (valid != false){
var options = {
numRows: 10,
@ -39,6 +39,7 @@ async function arrDepBoard(CRS){
var reply = await api.call("GetArrDepBoardWithDetails",options)
return reply
} else if (valid == false) {
log.out(`ldbService.arrDepBoard: Invalid 3ALPHA for lookup: ${CRS}`)
return {status: false, reason: "Invalid CRS/3alpha code"};
}
};

68
src/utils/dbinit.utils.js
View File

@ -1,57 +1,79 @@
// FUNCTIONS
// init() : Exported: Uses the internal functions to initialise databases.
// check() :
// build() : Adds CORPUS Data to DB.
// check() : Checks data presence and age.
// build() : Builds/Rebuilds collections.
const log = require('../utils/log.utils'); // Log Helper
const time = require('../utils/timeConvert.utils'); // Time Helper
const corpus = require('../services/corpus.services');
const dbAccess = require('../services/dbAccess.services');
async function init(){
var status = await check();
var status = await check('corpus');
if (status == "not_ready") {
try {
build("all")
build("corpus")
} catch (err) {
log.out("dbInit.init: Error building database")
log.out("dbInitUtils.init: Error building corpus database")
log.out(err)
}
}
var status = await check('stations')
if (status == "not_ready") {
try {
build("stations")
} catch (err) {
log.out("dbInitUtils.init: Error building stations database")
log.out(err)
}
}
}
async function check(){
log.out("DbInit.check: Checking database state")
let test = {'collection': 'any'};
async function check(coll){
log.out(`dbInitUtils.check: Checking collection '${coll}'`)
try {
await dbAccess.query('meta',test);
log.out(`DbInit.check: Reading Database Collection: meta`);
return "not_ready"
} catch (error) {
log.out(error)
var queryStr = {'type':'collection','target': coll};
var res = await dbAccess.query('meta',queryStr);
log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
var now = time.jsUnix(Date.now())
var delta = now - res['0']['updated']
} catch (err) {
log.out(`dbInitUtils.check: Unable to find out data age. Presume stale. Error Message:`)
log.out(err)
var delta = 12096000 // Extra zero to ensure data is updated.
}
var maxAge = 1209600 // 14 Days
if (delta > maxAge) {
log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds. Update pending`)
return "not_ready"
} else {
log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds. Update not required`)
return "ready"
}
}
async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`.
log.out("DbInit.build: Building database structure")
log.out("dbInitUtils.build: Building database structure")
var corpusAll = await corpus.get();
if (db === "corpus" || "all") {
if (db === "corpus") {
await dbAccess.dropCollection("corpus");
dbAccess.putCorpus(corpusAll);
meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
dbAccess.putMeta(meta);
log.out(`dbInitUtils.build: Updateing corpus meta`)
dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
}
if (db === "stations" || "all") {
if (db === "stations") {
await dbAccess.dropCollection("stations");
var corpusSubset = await corpus.subset(corpusAll);
dbAccess.putStations(corpusSubset);
meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}];
dbAccess.putMeta(meta);
log.out(`dbInitUtils.build: Updating stations meta`)
dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
}
}
module.exports = {
init,
check,
build
init
}

15
src/utils/timeConvert.utils.js Normal file
View File

@ -0,0 +1,15 @@
function unixLocal(unix) {
var jsTime = unix*1000
var dt = new Date(jsTime)
return dt.toLocaleString()
}
function jsUnix(js) {
var preRound = js / 1000
return Math.round(preRound)
}
module.exports = {
unixLocal,
jsUnix,
}