fred.boniface
/
OwlBoard
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

corpus.utils now working

This commit is contained in:
Fred Boniface 2022-12-08 16:39:13 +00:00
parent c4d668be81
commit ac86d9ee48
3 changed files with 43 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
package-lock.json generated
View File

@ -13,6 +13,7 @@
"express": "^4.18.2",
"ldbs-json": "^1.2.1",
"needle": "^3.2.0",
"node-gzip": "^1.1.2",
"sqlite3": "^5.1.2"
}
},
@ -1279,6 +1280,11 @@
"node": "^12.13.0 || ^14.15.0 || >=16.0.0"
}
},
"node_modules/node-gzip": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz",
"integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw=="
},
"node_modules/nopt": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
@ -2862,6 +2868,11 @@
}
}
},
"node-gzip": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz",
"integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw=="
},
"nopt": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",

1
package.json
View File

@ -4,6 +4,7 @@
"express": "^4.18.2",
"ldbs-json": "^1.2.1",
"needle": "^3.2.0",
"node-gzip": "^1.1.2",
"sqlite3": "^5.1.2"
},
"name": "owlboard",

55
src/utils/corpus.utils.js
View File

@ -2,50 +2,57 @@
// Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
// FUNCTIONS
// FUNCTIONS/
// init() : Exported: Uses the internal functions to return a clean CORPUS file.
// get() : Get the CORPUS data from Network Rail as a gzip file.
// extract(): Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary keys from the data.
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
const keys = require('/srv/keys/owlboard/keys.configs');
const axios = require('axios');
const zlib = require('zlib');
const keys = require('/srv/keys/owlboard/keys.configs')
const axios = require('axios')
const gz = require('node-gzip')
async function init() {
var gzipData = await get();
console.log(gzipData);
var rawData = await extract(gzipData);
console.log(rawData);
var cleanData = await clean(rawData);
console.log(cleanData)
return cleanData;
var gzipData = await get()
var corpus = await extract(gzipData)
console.log(corpus)
var cleanCorpus = await clean(corpus)
console.log(cleanCorpus)
return cleanCorpus
}
async function get() {
authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64');
authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64')
const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
const options = {
method: 'get',
timeout: 20000,
headers: {'Authorization': `Basic ${authHead}`},
responseType: 'arraybuffer'
};
var { output } = await axios.get(url, options)
console.log(output)
return output;
}
var { data } = await axios.get(url, options)
console.log(data)
return data
}
async function extract(input) {
var { data } = zlib.Gunzip(input);
let string = data.toString()
let obj = JSON.parse(string)
//console.log(obj);
return obj; // Find out how to get data out of callback -> See pinned tab on laptop2.ws
var raw = await gz.ungzip(input)
var obj = await JSON.parse(raw)
var output = obj.TIPLOCDATA
return output
}
async function clean() {
return "Empty Function: clean()"
async function clean(input) {
let clean = [];
for (const element of input) {
if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
delete(element.UIC);
delete(element.NLCDESC16);
delete(element.NLC);
clean.push(element);
}
}
return clean;
}
module.exports = {