Updated list options

This commit is contained in:
Fred Boniface 2022-12-21 00:42:32 +00:00
parent d330c3fddc
commit b0a6fcd522
5 changed files with 57 additions and 14 deletions

View File

@ -80,4 +80,18 @@ The app is designed to be run within Kubernetes or within a Docker container, as
In the case that OWL_LDB_SVKEY is not available, staff versions of departure board, etc. will not be available. In the case that OWL_LDB_SVKEY is not available, staff versions of departure board, etc. will not be available.
In the case that OWL_GIT_ISSUEBOT is not available, the 'Report Issue' page will not be able to POST data. In the case that OWL_GIT_ISSUEBOT is not available, the 'Report Issue' page will not be able to POST data.
## Database Layout
The OwlBoard application will build the database if required at startup. All it needs is authentication details for a MongoDB server.
### Collections
|Collection|Contents|Purpose|
|:--------:|:------:|:-----:|
|corpus|Raw CORPUS data with blank keys removed|Code lookups|
|stations|Cleaned CORPUS Data, any objects with blank 3ALPHA & STANOX fields are removed|Validation before fetching Arr/Dep boards|
|meta|Lists the update time of corpus and station data|Will be used to update after a predetermined time period|
Note that even after removing all objects from the CORPUS with a blank 3ALPHA & STANOX, many items remain which are not stations and will not have a board available. Going forwards methods to remove non-stations from this data will be introduced.

View File

@ -9,6 +9,16 @@ async function get(req, res, next){
} }
} }
async function getAll(req, res, next){
try {
res.json(await list.getAll(req.body))
} catch (err) {
console.erroe(`Unknown Error`, err.message);
next(err);
}
}
module.exports = { module.exports = {
get get,
getAll
} }

View File

@ -15,5 +15,6 @@ const listController = require('../controllers/list.controllers');
//router.delete('/:id', programmingLanguagesController.remove); //router.delete('/:id', programmingLanguagesController.remove);
router.get('/', listController.get); router.get('/', listController.get);
router.get('/all', listController.getAll);
module.exports = router; module.exports = router;

View File

@ -3,9 +3,15 @@ const corpus = require('../utils/corpus.utils');
async function get(){ async function get(){
// Databse lookup, get list of all stations as per the Python script output // Databse lookup, get list of all stations as per the Python script output
return corpus.init(); return corpus.initSubset();
}
async function getAll(){
// Raw corpus data
return corpus.initAll();
} }
module.exports = { module.exports = {
get get,
getAll
} }

View File

@ -3,10 +3,12 @@
// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js` // Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
// FUNCTIONS/ // FUNCTIONS/
// init() : Exported: Uses the internal functions to return a clean CORPUS file. // initSubset() : Exported: Uses the internal functions to return a clean CORPUS object.
// get() : Get the CORPUS data from Network Rail as a gzip file. // initAll() : Exported: Uses the internal functions to return a full CORPUS object.
// extract(): Extract the CORPUS JSON file from the GZIP file. // get() : Get the CORPUS data from Network Rail as a gzip file.
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data. // extract() : Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
// removeBlanks(): Removes keys with the value " ".
const axios = require('axios') const axios = require('axios')
const gz = require('node-gzip') const gz = require('node-gzip')
@ -14,13 +16,19 @@ const gz = require('node-gzip')
const corpusUser = process.env.OWL_LDB_CORPUSUSER const corpusUser = process.env.OWL_LDB_CORPUSUSER
const corpusPass = process.env.OWL_LDB_CORPUSPASS const corpusPass = process.env.OWL_LDB_CORPUSPASS
async function init() { async function initSubset() {
var gzipData = await get() var allCorpus = await initAll()
var corpus = await extract(gzipData) var cleanCorpus = await clean(allCorpus)
var cleanCorpus = await clean(corpus)
return cleanCorpus return cleanCorpus
} }
async function initAll(){
var gzipData = await get()
var allCorpus = await extract(gzipData)
var tidied = await removeBlanks(allCorpus)
return tidied
}
async function get() { async function get() {
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64') authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS' const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
@ -55,7 +63,11 @@ async function clean(input) {
return clean; return clean;
} }
async function removeBlanks(input){
return input;
}
module.exports = { module.exports = {
init, initAll,
outputToFile initSubset
} }