Begin automatic CORPUS download

app.js

@@ -11,6 +11,11 @@ const version  = require('./src/configs/version.configs');
 const testRtr  = require('./src/routes/test.routes');
 const listRtr  = require('./src/routes/list.routes');
 
+// TESTING ONLY
+const corpus = require('./src/utils/corpus.utils');
+
+corpus.getCorpus();
+
 // Print version number:
 console.log(`Starting OwlBoard - App Version: ${version.app} - API versions: ${version.api}`);
 

package-lock.json

@@ -11,6 +11,7 @@
       "dependencies": {
         "express": "^4.18.2",
         "ldbs-json": "^1.2.1",
+        "needle": "^3.2.0",
         "sqlite3": "^5.1.2"
       }
     },
@@ -1068,6 +1069,46 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     },
+    "node_modules/needle": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz",
+      "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==",
+      "dependencies": {
+        "debug": "^3.2.6",
+        "iconv-lite": "^0.6.3",
+        "sax": "^1.2.4"
+      },
+      "bin": {
+        "needle": "bin/needle"
+      },
+      "engines": {
+        "node": ">= 4.4.x"
+      }
+    },
+    "node_modules/needle/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/needle/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/needle/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
     "node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -2597,6 +2638,39 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
     },
+    "needle": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/needle/-/needle-3.2.0.tgz",
+      "integrity": "sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==",
+      "requires": {
+        "debug": "^3.2.6",
+        "iconv-lite": "^0.6.3",
+        "sax": "^1.2.4"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.7",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "iconv-lite": {
+          "version": "0.6.3",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+          "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        },
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
     "negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",

package.json

@@ -2,6 +2,7 @@
   "dependencies": {
     "express": "^4.18.2",
     "ldbs-json": "^1.2.1",
+    "needle": "^3.2.0",
     "sqlite3": "^5.1.2"
   },
   "name": "owlboard",

src/utils/corpus.utils.js

@@ -0,0 +1,32 @@
+// Get CORPUS data from Network Rail and format the data for OwlBoard
+
+// Username and password must be stored in `/srv/keys/owlboard/keys.config.js`
+
+const keys   = require('/srv/keys/owlboard/keys.configs');
+var https = require('https');
+
+// Need to add auth header `Basic {BASE64 HASH OF user:pass}
+
+async function getCorpus(){
+    url = `https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS`;
+    var request = https.request({'hostname': url}, 
+                                function (response) {
+                                    console.log('STATUS: ' + response.statusCode);
+                                    console.log('HEADERS: ' + JSON.stringify(response.headers));
+                                    response.setEncoding('utf8');
+                                    response.on('data', function (chunk) {
+                                    console.log('BODY: ' + chunk);
+                                    });
+                                });
+    request.end();
+}
+
+
+
+async function cleanCorpus(){
+
+}
+
+module.exports = {
+    getCorpus
+}

src/utils/dbinit.utils.js

@@ -1,3 +1,5 @@
 // Initialise database functions here
 // Rewrite Python3 script to clean CORPUS data
 // Also maybe scripting to FETCH the CORPUS data
+
+const corpus = require('../utils/corpus.utils');