diff --git a/app.js b/app.js index 3978886..35b9673 100644 --- a/app.js +++ b/app.js @@ -11,6 +11,11 @@ const version = require('./src/configs/version.configs'); const testRtr = require('./src/routes/test.routes'); const listRtr = require('./src/routes/list.routes'); +// TESTING ONLY +const corpus = require('./src/utils/corpus.utils'); + +corpus.getCorpus(); + // Print version number: console.log(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`); diff --git a/package-lock.json b/package-lock.json index 7acfabb..5f16720 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "express": "^4.18.2", "ldbs-json": "^1.2.1", + "needle": "^3.2.0", "sqlite3": "^5.1.2" } }, @@ -1068,6 +1069,46 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/needle": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz", + "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==", + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/needle/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/needle/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/needle/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -2597,6 +2638,39 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "needle": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz", + "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==", + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", diff --git a/package.json b/package.json index d1861d9..bea1daf 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "dependencies": { "express": "^4.18.2", "ldbs-json": "^1.2.1", + "needle": "^3.2.0", "sqlite3": "^5.1.2" }, "name": "owlboard", diff --git a/src/utils/corpus.utils.js b/src/utils/corpus.utils.js new file mode 100644 index 0000000..9f6a69a --- /dev/null +++ b/src/utils/corpus.utils.js @@ -0,0 +1,32 @@ +// Get CORPUS data from Network Rail and format the data for OwlBoard + +// Username and password must be stored in `/srv/keys/owlboard/keys.config.js` + +const keys = require('/srv/keys/owlboard/keys.configs'); +var https = require('https'); + +// Need to add auth header `Basic {BASE64 HASH OF user:pass} + +async function getCorpus(){ + url = `https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS`; + var request = https.request({'hostname': url}, + function (response) { + console.log('STATUS: ' + response.statusCode); + console.log('HEADERS: ' + JSON.stringify(response.headers)); + response.setEncoding('utf8'); + response.on('data', function (chunk) { + console.log('BODY: ' + chunk); + }); + }); + request.end(); +} + + + +async function cleanCorpus(){ + +} + +module.exports = { + getCorpus +} \ No newline at end of file diff --git a/src/utils/dbinit.utils.js b/src/utils/dbinit.utils.js index c142d2e..2cd393b 100644 --- a/src/utils/dbinit.utils.js +++ b/src/utils/dbinit.utils.js @@ -1,3 +1,5 @@ // Initialise database functions here // Rewrite Python3 script to clean CORPUS data -// Also maybe scripting to FETCH the CORPUS data \ No newline at end of file +// Also maybe scripting to FETCH the CORPUS data + +const corpus = require('../utils/corpus.utils'); \ No newline at end of file