fred.boniface/OwlBoard
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

Further edits to CORPUS

Browse Source
This commit is contained in:
Fred Boniface 2022-12-08 10:46:29 +00:00
parent 0d4b84c1ec
commit a00c7af2e0
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/utils/corpus.utils.js
View File

@ -2,28 +2,32 @@
// Username and password must be stored in `/srv/keys/owlboard/keys.config.js` // Username and password must be stored in `/srv/keys/owlboard/keys.config.js`
// FUNCTIONS
// init() : Exported: Uses the internal functions to return a clean CORPUS file.
// get() : Get the CORPUS data from Network Rail as a gzip file.
// extract(): Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary keys from the data.
const keys = require('/srv/keys/owlboard/keys.configs'); const keys = require('/srv/keys/owlboard/keys.configs');
const axios = require('axios'); const axios = require('axios');
const zlib = require('zlib'); const zlib = require('zlib');
async function initCorpus(){ // EXPORT - Uses getCorpus() to download the CORPUS GZIP and then processes it. async function initCorpus(){ // EXPORT - Uses getCorpus() to download the CORPUS GZIP and then processes it.
let gzipData = await getCorpus(); let gzipData = await getCorpusAgain();
console.log(gzipData); console.log(gzipData);
} }
async function getCorpus(){ // Download and return the GZIP file as an arraybuffer. async function getCorpus(){ // Download and return the GZIP file as an arraybuffer.
authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64'); authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64');
const getReq = axios.create({ const options = {
method: 'get', method: 'get',
baseURL: 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS', baseURL: 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS',
timeout: 3000, timeout: 3000,
headers: {'Authorization': `Basic ${authHead}`}, headers: {'Authorization': `Basic ${authHead}`},
responseType: 'arraybuffer' responseType: 'arraybuffer'
}); };
let download = await getReq().then(function (response){ let download = axios.get(options);
return response.data; console.log(download);
})
return download;
} }
async function getCorpusAgain(){ //Try to get CORPUS then pipe to zlib and return uncompressed date async function getCorpusAgain(){ //Try to get CORPUS then pipe to zlib and return uncompressed date
@ -47,7 +51,6 @@ async function getCorpusAgain(){ //Try to get CORPUS then pipe to zlib and retur
//console.log(obj); //console.log(obj);
return obj; // Find out how to get data out of callback -> See pinned tab on laptop2.ws return obj; // Find out how to get data out of callback -> See pinned tab on laptop2.ws
}) })
console.log(corpus); // This doesn't work but if console.logging from the function it prints.
} }
async function cleanCorpus(){ async function cleanCorpus(){
@ -56,5 +59,6 @@ async function cleanCorpus(){
module.exports = { module.exports = {
initCorpus, initCorpus,
getCorpusAgain getCorpusAgain,
getCorpus
} }