diff --git a/package-lock.json b/package-lock.json index fd22aa8..0e9c05e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "express": "^4.18.2", "ldbs-json": "^1.2.1", "needle": "^3.2.0", + "node-gzip": "^1.1.2", "sqlite3": "^5.1.2" } }, @@ -1279,6 +1280,11 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/node-gzip": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz", + "integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw==" + }, "node_modules/nopt": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", @@ -2862,6 +2868,11 @@ } } }, + "node-gzip": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz", + "integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw==" + }, "nopt": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", diff --git a/package.json b/package.json index 557b597..38f723a 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "express": "^4.18.2", "ldbs-json": "^1.2.1", "needle": "^3.2.0", + "node-gzip": "^1.1.2", "sqlite3": "^5.1.2" }, "name": "owlboard", diff --git a/src/utils/corpus.utils.js b/src/utils/corpus.utils.js index 2b28c44..69a2dc2 100644 --- a/src/utils/corpus.utils.js +++ b/src/utils/corpus.utils.js @@ -2,50 +2,57 @@ // Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js` -// FUNCTIONS +// FUNCTIONS/ // init() : Exported: Uses the internal functions to return a clean CORPUS file. // get() : Get the CORPUS data from Network Rail as a gzip file. // extract(): Extract the CORPUS JSON file from the GZIP file. -// clean() : Cleans the CORPUS data, removing unneccesary keys from the data. +// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data. -const keys = require('/srv/keys/owlboard/keys.configs'); -const axios = require('axios'); -const zlib = require('zlib'); +const keys = require('/srv/keys/owlboard/keys.configs') +const axios = require('axios') +const gz = require('node-gzip') async function init() { - var gzipData = await get(); - console.log(gzipData); - var rawData = await extract(gzipData); - console.log(rawData); - var cleanData = await clean(rawData); - console.log(cleanData) - return cleanData; + var gzipData = await get() + var corpus = await extract(gzipData) + console.log(corpus) + var cleanCorpus = await clean(corpus) + console.log(cleanCorpus) + return cleanCorpus } async function get() { - authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64'); + authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64') const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS' const options = { method: 'get', timeout: 20000, headers: {'Authorization': `Basic ${authHead}`}, responseType: 'arraybuffer' - }; - var { output } = await axios.get(url, options) - console.log(output) - return output; + } + var { data } = await axios.get(url, options) + console.log(data) + return data } async function extract(input) { - var { data } = zlib.Gunzip(input); - let string = data.toString() - let obj = JSON.parse(string) - //console.log(obj); - return obj; // Find out how to get data out of callback -> See pinned tab on laptop2.ws + var raw = await gz.ungzip(input) + var obj = await JSON.parse(raw) + var output = obj.TIPLOCDATA + return output } -async function clean() { - return "Empty Function: clean()" +async function clean(input) { + let clean = []; + for (const element of input) { + if (element.STANOX != ' ' && element['3ALPHA'] != ' '){ + delete(element.UIC); + delete(element.NLCDESC16); + delete(element.NLC); + clean.push(element); + } + } + return clean; } module.exports = {