fred.boniface/OwlBoard
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

CORPUS data is now stored in arraybuffer

Browse Source
This commit is contained in:
Fred Boniface 2022-12-07 13:22:00 +00:00
parent 1bdd18eb21
commit 7eac21411a
2 changed files with 20 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/services/ldb.services.js
View File

@ -2,7 +2,8 @@ const ldb = require('ldbs-json');
const keys = require('/srv/keys/owlboard/keys.configs'); const keys = require('/srv/keys/owlboard/keys.configs');
async function returnBoard(){ async function returnBoard(){
new ldb(keys.ldb,false) new ldb(keys.ldb,false);
}; };
module.exports = { module.exports = {

32
src/utils/corpus.utils.js
View File

@ -5,24 +5,26 @@
const keys = require('/srv/keys/owlboard/keys.configs'); const keys = require('/srv/keys/owlboard/keys.configs');
const axios = require('axios'); const axios = require('axios');
const zlib = require('zlib'); const zlib = require('zlib');
const fs = require('fs');
async function getCorpus(){ async function initCorpus(){ // EXPORT - Uses getCorpus() to download the CORPUS GZIP and then processes it.
let gzipData = await getCorpus();
console.log(gzipData);
}
async function getCorpus(){ // Download and return the GZIP file as an arraybuffer.
authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64'); authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64');
getOpts = { const getReq = axios.create({
method: 'get', method: 'get',
url: 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS', baseURL: 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS',
timeour: 3000, timeout: 3000,
headers: {'Authorization': `Basic ${authHead}`} headers: {'Authorization': `Basic ${authHead}`},
} responseType: 'arraybuffer'
axios(getOpts).then((response) => { });
decompress(response); let download = await getReq().then(function (response){
console.log(raw); return response.data;
}) })
} return download;
async function decompress(data){
let raw = zlib.gunzip(data);
return raw;
} }
async function cleanCorpus(){ async function cleanCorpus(){
@ -30,5 +32,5 @@ async function cleanCorpus(){
} }
module.exports = { module.exports = {
getCorpus initCorpus
} }