fred.boniface/OwlBoard
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects 1 Releases Wiki Activity

Begin automatic CORPUS download

Browse Source
This commit is contained in:
Fred Boniface 2022-12-06 01:20:43 +00:00
parent 40900bc502
commit d9d6e6c491
5 changed files with 115 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app.js
View File

@ -11,6 +11,11 @@ const version = require('./src/configs/version.configs');
const testRtr = require('./src/routes/test.routes');
const listRtr = require('./src/routes/list.routes');
// TESTING ONLY
const corpus = require('./src/utils/corpus.utils');
corpus.getCorpus();
// Print version number:
console.log(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);

74
package-lock.json generated
View File

@ -11,6 +11,7 @@
"dependencies": {
"express": "^4.18.2",
"ldbs-json": "^1.2.1",
"needle": "^3.2.0",
"sqlite3": "^5.1.2"
}
},
@ -1068,6 +1069,46 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/needle": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz",
"integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==",
"dependencies": {
"debug": "^3.2.6",
"iconv-lite": "^0.6.3",
"sax": "^1.2.4"
},
"bin": {
"needle": "bin/needle"
},
"engines": {
"node": ">= 4.4.x"
}
},
"node_modules/needle/node_modules/debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"dependencies": {
"ms": "^2.1.1"
}
},
"node_modules/needle/node_modules/iconv-lite": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/needle/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@ -2597,6 +2638,39 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"needle": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz",
"integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==",
"requires": {
"debug": "^3.2.6",
"iconv-lite": "^0.6.3",
"sax": "^1.2.4"
},
"dependencies": {
"debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"requires": {
"ms": "^2.1.1"
}
},
"iconv-lite": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
}
},
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",

1
package.json
View File

@ -2,6 +2,7 @@
"dependencies": {
"express": "^4.18.2",
"ldbs-json": "^1.2.1",
"needle": "^3.2.0",
"sqlite3": "^5.1.2"
},
"name": "owlboard",

32
src/utils/corpus.utils.js Normal file
View File

@ -0,0 +1,32 @@
// Get CORPUS data from Network Rail and format the data for OwlBoard
// Username and password must be stored in `/srv/keys/owlboard/keys.config.js`
const keys = require('/srv/keys/owlboard/keys.configs');
var https = require('https');
// Need to add auth header `Basic {BASE64 HASH OF user:pass}
async function getCorpus(){
url = `https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS`;
var request = https.request({'hostname': url},
function (response) {
console.log('STATUS: ' + response.statusCode);
console.log('HEADERS: ' + JSON.stringify(response.headers));
response.setEncoding('utf8');
response.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
request.end();
}
async function cleanCorpus(){
}
module.exports = {
getCorpus
}

4
src/utils/dbinit.utils.js
View File

@ -1,3 +1,5 @@
// Initialise database functions here
// Rewrite Python3 script to clean CORPUS data
// Also maybe scripting to FETCH the CORPUS data
// Also maybe scripting to FETCH the CORPUS data
const corpus = require('../utils/corpus.utils');