OwlBoard/backend
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Remove Static, Remove DbInit, Adjust awaits, increment version

Browse Source 
Signed-off-by: Fred Boniface <fred@fjla.uk>
This commit is contained in:
Fred Boniface
2023-02-16 21:25:19 +00:00
parent b1940bcfbb
commit d10d558715
88 changed files with 1019 additions and 4224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/configs/version.configs.js
View File

@@ -1,6 +1,6 @@
const version = {
api: ["/api/v1/",],
app: "0.1.1"
app: "1.0.0"
};
module.exports = version;

73
src/services/corpus.services.js
View File

@@ -1,73 +0,0 @@
// Get CORPUS data from Network Rail and format the data for OwlBoard
// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
// FUNCTIONS/
// initSubset() : Exported: Uses the internal functions to return a clean CORPUS object.
// initAll() : Exported: Uses the internal functions to return a full CORPUS object.
// get() : Get the CORPUS data from Network Rail as a gzip file.
// extract() : Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary non-stations from the data.
const log = require('../utils/log.utils'); // Log Helper
const axios = require('axios')
const gz = require('node-gzip')
const corpusUser = process.env.OWL_LDB_CORPUSUSER
const corpusPass = process.env.OWL_LDB_CORPUSPASS
async function subset(allCorpus) {
return (await clean(allCorpus))
}
async function get() {
var gzipData = await fetch()
return (await extract(gzipData))
}
async function fetch() {
log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
const url = 'https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
const options = {
method: 'get',
timeout: 20000,
headers: {'Authorization': `Basic ${authHead}`},
responseType: 'arraybuffer'
}
try {
var { data } = await axios.get(url, options)
log.out("corpus.fetch: CORPUS Data fetched")
} catch (error) {
log.out("corpus.fetch: Error fetching CORPUS")
log.out(error)
}
return data
}
async function extract(input) {
log.out(`corpus.extract: Extracting CORPUS archive`)
var raw = await gz.ungzip(input)
var obj = await JSON.parse(raw)
return (obj.TIPLOCDATA)
}
async function clean(input) {
log.out(`corpus.clean: Removing non-stations from CORPUS data`)
let clean = [];
for (const element of input) {
if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
delete(element.UIC);
delete(element.NLCDESC16);
delete(element.NLC);
clean.push(element);
}
}
return clean;
}
module.exports = {
get,
subset
}

96
src/services/dbAccess.services.js
View File

@@ -12,62 +12,6 @@ const { MongoClient } = require('mongodb');
const client = new MongoClient(uri);
const db = client.db(dbName);
async function dropCollection(coll){
await client.connect();
// check if collection contains any documents, if it doesn't, it is either empty or non-existent - it doesn't need dropping.
var collection = db.collection(coll);
var count = await collection.countDocuments();
log.out(`DbAccess.dropCollection: Collection '${coll}' contains ${count} documents`)
if (count == 0) {
log.out(`DbAccess.dropCollection: Collection '${coll}' is empty. Do not need to drop`)
} else {
log.out(`DbAccess.dropCollection: dropping collection: '${coll}'`)
db.dropCollection(coll);
log.out(`DbAccess.dropCollection: dropped collection: '${coll}'`)
}
}
async function putCorpus(data){
log.out("DbAccess.putCorpus: Uploading CORPUS data to database")
await client.connect();
try {
var coll = db.collection("corpus");
await coll.insertMany(data);
} catch (error) {
log.out("DbAccess.putCorpus: Error uploading Corpus data to database")
log.out(error)
}
};
async function putStations(data){
log.out("DbAccess.putStations: Uploading Stations data to database")
await client.connect();
try {
var coll = db.collection("stations");
coll.insertMany(data);
} catch (error) {
log.out("DbAccess.putStations: Error uploading Stations data to database")
log.out(error)
}
};
async function updateMeta(type, target, unixTime){
await client.connect();
var coll = db.collection("meta");
var filter = {type: type, target: target};
var update = {$set:{updated: unixTime}};
var options = {upsert: true}; // If document isn't present will insert.
try {
var result = await coll.updateOne(filter,update,options)
log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
log.out(`dbAccessServices.updateMeta: meta for '${target}' updated`)
} catch (err) {
log.out(`dbAccessServices.updateMeta: Unable to update meta for '${target}'`)
log.out(err)
}
}
async function query(collection, query){
await client.connect();
log.out(`dbAccess.query: Connecting to collection: '${collection}'`)
@@ -79,22 +23,8 @@ async function query(collection, query){
return (await qcursor.toArray());
}
async function ensureIndex(col, field, text) {
await client.connect();
if (!text) {
log.out(`dbAccess.ensureIndex: Creating index in collection ${col} for field ${field}`)
db.createIndex(col, field);
} else {
log.out(`dbAccess.ensureIndex: Creating text index in collection ${col} for field ${field}`)
let idx = {}
idx[field] = "text";
db.createIndex(col, idx);
}
log.out(`dbAccess.ensureIndex: Index created`);
return;
}
async function increment(target) {
log.out(`dbAccess.increment: Incrementing counter for: ${target}`)
await client.connect();
let col = db.collection("meta");
let update = {}
@@ -103,29 +33,7 @@ async function increment(target) {
return;
}
async function createCount() {
await client.connect();
let col = db.collection("meta");
var filter = {type: "count", target: "counters"};
var update = {$set:{/*since: new Date,*/ type: "count", target: "counters"}};
var options = {upsert: true}; // If document isn't present will insert.
try {
var result = await col.updateOne(filter,update,options)
log.out(`dbAccessServices.updateMeta: ${JSON.stringify(result)}`)
log.out(`dbAccessServices.updateMeta: count meta added updated`)
} catch (err) {
log.out(`dbAccessServices.updateMeta: Unable to add count`)
log.out(err)
}
}
module.exports = {
putCorpus,
putStations,
dropCollection,
updateMeta,
query,
ensureIndex,
increment,
createCount
increment
}

16
src/services/ldb.services.js
View File

@@ -1,12 +1,5 @@
// Parse and return an LDB Request
// FUNCTIONS
// post(body, id): Exported:
// body: [req.body from controller]
// id : [req.params.id from controller - this is expected to be CRS or TIPLOC]
// convertTiploc(TIPLOC) : Exported: Looks up CRS, Name & STANOX for Tiploc
const log = require('../utils/log.utils'); // Log Helper
const ldb = require('ldbs-json')
const util = require('../utils/ldb.utils')
@@ -22,8 +15,9 @@ async function get(body, id){
try {
var crs = obj[0]['3ALPHA'];
log.out(`ldbService.get: Determined CRS for lookup to be: ${crs}`);
var data = await arrDepBoard(crs);
db.increment("ldbws") // Need to add creation of this document to the database. >> {type:"count",counting:"api_hit",target:"ldbws",since:"DATE"}
var data = arrDepBoard(crs);
db.increment("ldbws");
await data;
} catch (err) {
log.out(`ldbService.get: Error, Unable to find CRS: ${err}`)
var data = {ERROR:'NOT_FOUND',description:'The entered station was not found. Please check and try again.'};
@@ -39,8 +33,8 @@ async function arrDepBoard(CRS){
crs: CRS.toUpperCase()
}
var api = new ldb(ldbKey,false)
var reply = await api.call("GetArrDepBoardWithDetails",options)
return reply
var reply = api.call("GetArrDepBoardWithDetails", options, false, false)
return await reply
} catch (err) {
log.out(`ldbService.arrDepBoard: Lookup Failed for: ${CRS}`)
return {GetStationBoardResult: "not available", Reason: `The CRS code ${CRS} is not valid`, Why: `Sometimes a station will have more than one CRS - for example Filton Abbey Wood has FIT and FAW however schedules are only available when looking up with FIT - this is how the National Rail Enquiries systems work.`};

12
src/services/list.services.js
View File

@@ -3,15 +3,15 @@ const db = require('../services/dbAccess.services')
const os = require('os')
async function getStations(){
var out = await db.query("stations")
log.out(`listServices.getStations: fetched stations list`)
return out;
var out = db.query("stations")
log.out(`listServices.getStations: Fetching stations list`)
return await out;
}
async function getCorpus(){
var out = await db.query("corpus")
log.out(`listServices.getCorpus: fetched CORPUS list`)
return out;
var out = db.query("corpus")
log.out(`listServices.getCorpus: Fetching CORPUS list`)
return await out;
}
module.exports = {

90
src/utils/DEP_dbinit.utils.js
View File

@@ -1,90 +0,0 @@
// FUNCTIONS
// init() : Exported: Uses the internal functions to initialise databases.
// check() : Checks data presence and age.
// build() : Builds/Rebuilds collections.
const log = require('./log.utils'); // Log Helper
const time = require('./timeConvert.utils'); // Time Helper
const corpus = require('../services/corpus.services');
const dbAccess = require('../services/dbAccess.services');
async function init(){
var status = await check('corpus');
if (status == "not_ready") {
try {
await build("corpus")
} catch (err) {
log.out("dbInitUtils.init: Error building corpus database")
log.out(err)
}
}
var status = await check('stations')
if (status == "not_ready") {
try {
await build("stations")
} catch (err) {
log.out("dbInitUtils.init: Error building stations database")
log.out(err)
}
}
indexes();
dbAccess.createCount();
}
async function check(coll){
log.out(`dbInitUtils.check: Checking collection '${coll}'`)
try {
var queryStr = {'type':'collection','target': coll};
var res = await dbAccess.query('meta',queryStr);
log.out(`dbInitUtils.check: Last update of ${coll}: ${time.unixLocal(res['0']['updated'])}`)
var now = time.jsUnix(Date.now())
var delta = now - res['0']['updated']
} catch (err) {
log.out(`dbInitUtils.check: Unable to find out data age. Presume stale. Error Message:`)
log.out(err)
var delta = 12096000 // Extra zero to ensure data is updated.
}
var maxAge = 1209600 // 14 Days
if (delta > maxAge) {
log.out(`dbInitUtils.check: '${coll}' data older than max age ${maxAge} seconds. Update pending`)
return "not_ready"
} else {
log.out(`dbInitUtils.check: '${coll}' data newer than max age ${maxAge} seconds. Update not required`)
return "ready"
}
}
async function build(db){ // `db` must be one of: `corpus`, `stations`, `all`.
log.out("dbInitUtils.build: Building database structure")
var corpusAll = await corpus.get();
if (db === "corpus") {
await dbAccess.dropCollection("corpus");
dbAccess.putCorpus(corpusAll);
log.out(`dbInitUtils.build: Updating corpus meta`);
dbAccess.updateMeta("collection", "corpus", time.jsUnix(Date.now()));
}
if (db === "stations") {
await dbAccess.dropCollection("stations");
var corpusSubset = await corpus.subset(corpusAll);
dbAccess.putStations(corpusSubset);
log.out(`dbInitUtils.build: Updating stations meta`);
dbAccess.updateMeta("collection", "stations", time.jsUnix(Date.now()));
}
}
async function indexes() {
dbAccess.ensureIndex("corpus", "NLC");
dbAccess.ensureIndex("corpus", "3ALPHA");
dbAccess.ensureIndex("stations", "3ALPHA");
dbAccess.ensureIndex("stations", "STANOX");
dbAccess.ensureIndex("stations", "TIPLOC");
dbAccess.ensureIndex("corpus", "NLCDESC", "text")
}
module.exports = {
init
}