73 lines
2.3 KiB
JavaScript
73 lines
2.3 KiB
JavaScript
// Get CORPUS data from Network Rail and format the data for OwlBoard
|
|
|
|
// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
|
|
|
|
// FUNCTIONS/
|
|
// initSubset() : Exported: Uses the internal functions to return a clean CORPUS object.
|
|
// initAll() : Exported: Uses the internal functions to return a full CORPUS object.
|
|
// get() : Get the CORPUS data from Network Rail as a gzip file.
|
|
// extract() : Extract the CORPUS JSON file from the GZIP file.
|
|
// clean() : Cleans the CORPUS data, removing unneccesary non-stations from the data.
|
|
|
|
const log = require('../utils/log.utils'); // Log Helper
|
|
|
|
const axios = require('axios')
|
|
const gz = require('node-gzip')
|
|
|
|
const corpusUser = process.env.OWL_LDB_CORPUSUSER
|
|
const corpusPass = process.env.OWL_LDB_CORPUSPASS
|
|
|
|
async function subset(allCorpus) {
|
|
return (await clean(allCorpus))
|
|
}
|
|
|
|
async function get() {
|
|
var gzipData = await fetch()
|
|
return (await extract(gzipData))
|
|
}
|
|
|
|
async function fetch() {
|
|
log.out("corpus.fetch: Fetching CORPUS Data from Network Rail")
|
|
authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
|
|
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
|
|
const options = {
|
|
method: 'get',
|
|
timeout: 20000,
|
|
headers: {'Authorization': `Basic ${authHead}`},
|
|
responseType: 'arraybuffer'
|
|
}
|
|
try {
|
|
var { data } = await axios.get(url, options)
|
|
log.out("corpus.fetch: CORPUS Data fetched")
|
|
} catch (error) {
|
|
log.out("corpus.fetch: Error fetching CORPUS")
|
|
log.out(error)
|
|
}
|
|
return data
|
|
}
|
|
|
|
async function extract(input) {
|
|
log.out(`corpus.extract: Extracting CORPUS archive`)
|
|
var raw = await gz.ungzip(input)
|
|
var obj = await JSON.parse(raw)
|
|
return (obj.TIPLOCDATA)
|
|
}
|
|
|
|
async function clean(input) {
|
|
log.out(`corpus.clean: Removing non-stations from CORPUS data`)
|
|
let clean = [];
|
|
for (const element of input) {
|
|
if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
|
|
delete(element.UIC);
|
|
delete(element.NLCDESC16);
|
|
delete(element.NLC);
|
|
clean.push(element);
|
|
}
|
|
}
|
|
return clean;
|
|
}
|
|
|
|
module.exports = {
|
|
get,
|
|
subset
|
|
} |