Updated list options

This commit is contained in:
Fred Boniface 2022-12-21 00:42:32 +00:00
parent d330c3fddc
commit b0a6fcd522
5 changed files with 57 additions and 14 deletions

View File

@ -81,3 +81,17 @@ The app is designed to be run within Kubernetes or within a Docker container, as
In the case that OWL_LDB_SVKEY is not available, staff versions of departure board, etc. will not be available.
In the case that OWL_GIT_ISSUEBOT is not available, the 'Report Issue' page will not be able to POST data.
## Database Layout
The OwlBoard application will build the database if required at startup. All it needs is authentication details for a MongoDB server.
### Collections
|Collection|Contents|Purpose|
|:--------:|:------:|:-----:|
|corpus|Raw CORPUS data with blank keys removed|Code lookups|
|stations|Cleaned CORPUS Data, any objects with blank 3ALPHA & STANOX fields are removed|Validation before fetching Arr/Dep boards|
|meta|Lists the update time of corpus and station data|Will be used to update after a predetermined time period|
Note that even after removing all objects from the CORPUS with a blank 3ALPHA & STANOX, many items remain which are not stations and will not have a board available. Going forwards methods to remove non-stations from this data will be introduced.

View File

@ -9,6 +9,16 @@ async function get(req, res, next){
}
}
module.exports = {
get
async function getAll(req, res, next){
try {
res.json(await list.getAll(req.body))
} catch (err) {
console.erroe(`Unknown Error`, err.message);
next(err);
}
}
module.exports = {
get,
getAll
}

View File

@ -15,5 +15,6 @@ const listController = require('../controllers/list.controllers');
//router.delete('/:id', programmingLanguagesController.remove);
router.get('/', listController.get);
router.get('/all', listController.getAll);
module.exports = router;

View File

@ -3,9 +3,15 @@ const corpus = require('../utils/corpus.utils');
async function get(){
// Databse lookup, get list of all stations as per the Python script output
return corpus.init();
return corpus.initSubset();
}
async function getAll(){
// Raw corpus data
return corpus.initAll();
}
module.exports = {
get
get,
getAll
}

View File

@ -3,10 +3,12 @@
// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
// FUNCTIONS/
// init() : Exported: Uses the internal functions to return a clean CORPUS file.
// initSubset() : Exported: Uses the internal functions to return a clean CORPUS object.
// initAll() : Exported: Uses the internal functions to return a full CORPUS object.
// get() : Get the CORPUS data from Network Rail as a gzip file.
// extract() : Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
// removeBlanks(): Removes keys with the value " ".
const axios = require('axios')
const gz = require('node-gzip')
@ -14,13 +16,19 @@ const gz = require('node-gzip')
const corpusUser = process.env.OWL_LDB_CORPUSUSER
const corpusPass = process.env.OWL_LDB_CORPUSPASS
async function init() {
var gzipData = await get()
var corpus = await extract(gzipData)
var cleanCorpus = await clean(corpus)
async function initSubset() {
var allCorpus = await initAll()
var cleanCorpus = await clean(allCorpus)
return cleanCorpus
}
async function initAll(){
var gzipData = await get()
var allCorpus = await extract(gzipData)
var tidied = await removeBlanks(allCorpus)
return tidied
}
async function get() {
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
@ -55,7 +63,11 @@ async function clean(input) {
return clean;
}
module.exports = {
init,
outputToFile
async function removeBlanks(input){
return input;
}
module.exports = {
initAll,
initSubset
}