From 17ea61d35bf4b18c5fd69c60257480095aee91ac Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Thu, 5 Jan 2023 21:00:39 +0000 Subject: [PATCH] Update log methods --- app.js | 21 ++++++++++++------- src/services/corpus.services.js | 13 +++++++++++- src/services/dbAccess.services.js | 35 ++++++++++++++++++++++--------- src/services/ldb.services.js | 3 +++ src/utils/dbinit.utils.js | 24 ++++++++++++++------- src/utils/ldb.utils.js | 2 ++ src/utils/log.utils.js | 8 +++++++ 7 files changed, 80 insertions(+), 26 deletions(-) create mode 100644 src/utils/log.utils.js diff --git a/app.js b/app.js index 580072a..d64e507 100644 --- a/app.js +++ b/app.js @@ -9,18 +9,19 @@ const express = require('express'); const app = express(); // Internal Requires -const version = require('./src/configs/version.configs'); -const testRtr = require('./src/routes/test.routes'); -const listRtr = require('./src/routes/list.routes'); -const ldbRtr = require('./src/routes/ldb.routes'); -const kubeRtr = require('./src/routes/kube.routes'); +const log = require('./src/utils/log.utils'); // Log Helper +const version = require('./src/configs/version.configs'); // Version Strings +const listRtr = require('./src/routes/list.routes'); // /list endpoints +const ldbRtr = require('./src/routes/ldb.routes'); // /ldb endpoints +const kubeRtr = require('./src/routes/kube.routes'); // /kube endpoints +const initDb = require('./src/utils/dbinit.utils') // DB Init Utility // Set Server Configurations const srvListen = process.env.OWL_SRV_LISTEN || "0.0.0.0" const srvPort = process.env.OWL_SRV_PORT || 8460 // Print version number: -console.log(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); +log.out(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); // Test for required vars: // const varTest = require('./src/utils/varTest.utils'); @@ -31,6 +32,9 @@ console.log(`Starting OwlBoard - App Version: ${version.app} - API versions: ${v // console.log("Unable to start, missing required vars") // exit app +// DB Init +initDb.check(); + // Express Error Handling: app.use((err, req, res, next) => { const statusCode = err.statuscode || 500; @@ -51,8 +55,9 @@ app.use('/api/v1/kube', kubeRtr); // Start Express app.listen(srvPort, srvListen, (error) =>{ if(!error) { - console.log(`Started server on http://${srvListen}:${srvPort}`); + log.out(`Server: Started on http://${srvListen}:${srvPort}`); + log.out("State: alive") } else { - console.log("Error occurred, server can't start", error); + log.out("Error occurred, server can't start", error); } }); \ No newline at end of file diff --git a/src/services/corpus.services.js b/src/services/corpus.services.js index 0b660ec..71cbd54 100644 --- a/src/services/corpus.services.js +++ b/src/services/corpus.services.js @@ -9,6 +9,8 @@ // extract() : Extract the CORPUS JSON file from the GZIP file. // clean() : Cleans the CORPUS data, removing unneccesary non-stations from the data. +const log = require('../utils/log.utils'); // Log Helper + const axios = require('axios') const gz = require('node-gzip') @@ -27,6 +29,7 @@ async function get(){ } async function fetch() { + log.out("Corpus: Fetching CORPUS Data from Network Rail") authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64') const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS' const options = { @@ -35,11 +38,18 @@ async function fetch() { headers: {'Authorization': `Basic ${authHead}`}, responseType: 'arraybuffer' } - var { data } = await axios.get(url, options) + try { + var { data } = await axios.get(url, options) + log.out("Corpus: CORPUS Data fetched") + } catch (error) { + log.out("Corpus: Error fetching CORPUS") + log.out(error) + } return data } async function extract(input) { + log.out(`Corpus: Extracting CORPUS archive`) var raw = await gz.ungzip(input) var obj = await JSON.parse(raw) var output = obj.TIPLOCDATA @@ -47,6 +57,7 @@ async function extract(input) { } async function clean(input) { + log.out(`Corpus: Removing non-stations from CORPUS data`) let clean = []; for (const element of input) { if (element.STANOX != ' ' && element['3ALPHA'] != ' '){ diff --git a/src/services/dbAccess.services.js b/src/services/dbAccess.services.js index 49ef624..ace2325 100644 --- a/src/services/dbAccess.services.js +++ b/src/services/dbAccess.services.js @@ -1,3 +1,5 @@ +const log = require('../utils/log.utils'); // Log Helper + const dbUser = process.env.OWL_DB_USER || "owl" const dbPass = process.env.OWL_DB_PASS || "twittwoo" const dbName = process.env.OWL_DB_NAME || "owlboard" @@ -10,39 +12,52 @@ const { MongoClient } = require('mongodb'); const client = new MongoClient(uri); const db = client.db(dbName); + async function putCorpus(data){ + log.out("DbAccess: Uploading Corpus data to database") await client.connect(); var coll = db.collection("corpus"); - var result = coll.insertMany(data); - console.log(result); + try { + await coll.insertMany(data); + log.out("DbAccess: Corpus upload complete") + } catch (error) { + log.out("DbAccess: Error uploading Corpus data to database") + log.out(error) + } }; async function putStations(data){ + log.out("DbAccess: Uploading Stations data to database") await client.connect(); var coll = db.collection("stations"); - var result = coll.insertMany(data); - console.log(result); + try { + coll.insertMany(data); + log.out("DbAccess: Stations upload complete") + } catch (error) { + log.out("DbAccess: Error uploading Stations data to database") + log.out(error) + } }; async function putMeta(data){ await client.connect(); var coll = db.collection("meta"); - var result = coll.insertMany(data); - console.log(result); + coll.insertMany(data); } //async function putData(data,collection) { // await client.connect() // var coll = db.collection(collection); // var result = coll.insertMany(data); -// console.log(result); +// log.out(result); //}; async function query(collection, query){ await client.connect(); - var coll = db.collection(collection); - var result = coll.find(query) //??? - + let qcoll = db.collection(collection); + let qcursor = qcoll.find(query) + let qresult = await qcursor.toArray(); + return qresult; } module.exports = { diff --git a/src/services/ldb.services.js b/src/services/ldb.services.js index 5e87e87..9932959 100644 --- a/src/services/ldb.services.js +++ b/src/services/ldb.services.js @@ -7,6 +7,8 @@ // convertTiploc(TIPLOC) : Exported: Looks up CRS, Name & STANOX for Tiploc +const log = require('../utils/log.utils'); // Log Helper + const ldb = require('ldbs-json') const util = require('../utils/ldb.utils') @@ -27,6 +29,7 @@ async function get(body, id){ async function arrDepBoard(CRS){ var valid = await util.checkCrs(CRS) + log.out(`ldbService: Fetching ArrDep Board for ${CRS}`) if (valid != false){ var options = { numRows: 10, diff --git a/src/utils/dbinit.utils.js b/src/utils/dbinit.utils.js index 95c5568..451561f 100644 --- a/src/utils/dbinit.utils.js +++ b/src/utils/dbinit.utils.js @@ -3,6 +3,7 @@ // check() : // build() : Adds CORPUS Data to DB. +const log = require('../utils/log.utils'); // Log Helper const corpus = require('../services/corpus.services'); const dbAccess = require('../services/dbAccess.services'); @@ -12,27 +13,36 @@ async function init(){ } async function check(){ - // Check meta collection for last update - // If last update more than 14 days () ago then return false - // Check corpus and stations collections exist. If not return false - // Return true if checks pass. + log.out("DbInit: Checking database state") + let test = {'collection': 'any'}; + try { + let meta = await dbAccess.query('meta',test); + log.out(`DbInit: Reading Database Collection: meta`); + } catch (error) {log.out(error)} + if (false){ + log.out("DbInit: Database structure not initialised") + build("all") + } } async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`. + log.out("DbInit: Building database structure") var corpusAll = await corpus.get(); if (db === "corpus" || "all") { dbAccess.putCorpus(corpusAll); - meta = {collection:"any",updated:Math.floor(Date.now() / 1000)}; + meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}]; dbAccess.putMeta(meta); } if (db === "stations" || "all") { var corpusSubset = await corpus.subset(corpusAll); dbAccess.putStations(corpusSubset); - meta = {collection:"any",updated:Math.floor(Date.now() / 1000)}; + meta = [{collection:"any",updated:Math.floor(Date.now() / 1000)}]; dbAccess.putMeta(meta); } } module.exports = { - init + init, + check, + build } \ No newline at end of file diff --git a/src/utils/ldb.utils.js b/src/utils/ldb.utils.js index 61e7f53..11d64bb 100644 --- a/src/utils/ldb.utils.js +++ b/src/utils/ldb.utils.js @@ -1,3 +1,5 @@ +const log = require('../utils/log.utils'); // Log Helper + async function checkCrs(input){ // Check whether CRS is valid // if not, try to get tiploc diff --git a/src/utils/log.utils.js b/src/utils/log.utils.js new file mode 100644 index 0000000..a79b8e8 --- /dev/null +++ b/src/utils/log.utils.js @@ -0,0 +1,8 @@ +function out(msg) { + var time = new Date().toISOString(); + console.log(`${time} - ${msg}`) +} + +module.exports = { + out +} \ No newline at end of file