Add data normalisation and remove hash libraries & functions as hashes are no longer required.

package-lock.json

@@ -12,14 +12,11 @@
         "@aws-sdk/client-s3": "^3.964.0",
         "@aws-sdk/lib-storage": "^3.964.0",
         "@owlboard/backend-data-contracts": "^0.1.9",
-        "mongodb": "^7.0.0",
         "nats": "^2.29.3",
-        "readline": "^1.3.0",
+        "readline": "^1.3.0"
-        "xxhashjs": "^0.2.2"
       },
       "devDependencies": {
         "@types/node": "^25.0.3",
-        "@types/xxhashjs": "^0.2.4",
         "tsx": "^4.21.0",
         "typescript": "^5.9.3"
       }
@@ -1385,15 +1382,6 @@
         "node": ">=18"
       }
     },
-    "node_modules/@mongodb-js/saslprep": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.4.tgz",
-      "integrity": "sha512-p7X/ytJDIdwUfFL/CLOhKgdfJe1Fa8uw9seJYvdOmnP9JBWGWHW69HkOixXS6Wy9yvGf1MbhcS6lVmrhy4jm2g==",
-      "license": "MIT",
-      "dependencies": {
-        "sparse-bitfield": "^3.0.3"
-      }
-    },
     "node_modules/@owlboard/backend-data-contracts": {
       "version": "0.1.9",
       "resolved": "https://git.fjla.uk/api/packages/OwlBoard/npm/%40owlboard%2Fbackend-data-contracts/-/0.1.9/backend-data-contracts-0.1.9.tgz",
@@ -2142,31 +2130,6 @@
         "undici-types": "~7.16.0"
       }
     },
-    "node_modules/@types/webidl-conversions": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
-      "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==",
-      "license": "MIT"
-    },
-    "node_modules/@types/whatwg-url": {
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz",
-      "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/webidl-conversions": "*"
-      }
-    },
-    "node_modules/@types/xxhashjs": {
-      "version": "0.2.4",
-      "resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.4.tgz",
-      "integrity": "sha512-E2+ZoJY2JjmVPN0iQM5gJvZkk98O2PYXSi6HrciEk3EKF34+mauEk/HgwTeCz+2r8HXHMKpucrwy4qTT12OPaQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -2193,15 +2156,6 @@
       "integrity": "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==",
       "license": "MIT"
     },
-    "node_modules/bson": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/bson/-/bson-7.0.0.tgz",
-      "integrity": "sha512-Kwc6Wh4lQ5OmkqqKhYGKIuELXl+EPYSCObVE6bWsp1T/cGkOCBN0I8wF/T44BiuhHyNi1mmKVPXk60d41xZ7kw==",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=20.19.0"
-      }
-    },
     "node_modules/buffer": {
       "version": "5.6.0",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
@@ -2212,12 +2166,6 @@
         "ieee754": "^1.1.4"
       }
     },
-    "node_modules/cuint": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
-      "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==",
-      "license": "MIT"
-    },
     "node_modules/esbuild": {
       "version": "0.27.2",
       "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz",
@@ -2341,71 +2289,6 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
       "license": "ISC"
     },
-    "node_modules/memory-pager": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
-      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
-      "license": "MIT"
-    },
-    "node_modules/mongodb": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.0.0.tgz",
-      "integrity": "sha512-vG/A5cQrvGGvZm2mTnCSz1LUcbOPl83hfB6bxULKQ8oFZauyox/2xbZOoGNl+64m8VBrETkdGCDBdOsCr3F3jg==",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "@mongodb-js/saslprep": "^1.3.0",
-        "bson": "^7.0.0",
-        "mongodb-connection-string-url": "^7.0.0"
-      },
-      "engines": {
-        "node": ">=20.19.0"
-      },
-      "peerDependencies": {
-        "@aws-sdk/credential-providers": "^3.806.0",
-        "@mongodb-js/zstd": "^7.0.0",
-        "gcp-metadata": "^7.0.1",
-        "kerberos": "^7.0.0",
-        "mongodb-client-encryption": ">=7.0.0 <7.1.0",
-        "snappy": "^7.3.2",
-        "socks": "^2.8.6"
-      },
-      "peerDependenciesMeta": {
-        "@aws-sdk/credential-providers": {
-          "optional": true
-        },
-        "@mongodb-js/zstd": {
-          "optional": true
-        },
-        "gcp-metadata": {
-          "optional": true
-        },
-        "kerberos": {
-          "optional": true
-        },
-        "mongodb-client-encryption": {
-          "optional": true
-        },
-        "snappy": {
-          "optional": true
-        },
-        "socks": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/mongodb-connection-string-url": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.0.tgz",
-      "integrity": "sha512-irhhjRVLE20hbkRl4zpAYLnDMM+zIZnp0IDB9akAFFUZp/3XdOfwwddc7y6cNvF2WCEtfTYRwYbIfYa2kVY0og==",
-      "license": "Apache-2.0",
-      "dependencies": {
-        "@types/whatwg-url": "^13.0.0",
-        "whatwg-url": "^14.1.0"
-      },
-      "engines": {
-        "node": ">=20.19.0"
-      }
-    },
     "node_modules/nats": {
       "version": "2.29.3",
       "resolved": "https://registry.npmjs.org/nats/-/nats-2.29.3.tgz",
@@ -2430,15 +2313,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/punycode": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/readable-stream": {
       "version": "3.6.2",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
@@ -2489,15 +2363,6 @@
       ],
       "license": "MIT"
     },
-    "node_modules/sparse-bitfield": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
-      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
-      "license": "MIT",
-      "dependencies": {
-        "memory-pager": "^1.0.2"
-      }
-    },
     "node_modules/stream-browserify": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz",
@@ -2529,18 +2394,6 @@
       ],
       "license": "MIT"
     },
-    "node_modules/tr46": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
-      "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
-      "license": "MIT",
-      "dependencies": {
-        "punycode": "^2.3.1"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    },
     "node_modules/tslib": {
       "version": "2.8.1",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
@@ -2599,37 +2452,6 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
       "license": "MIT"
-    },
-    "node_modules/webidl-conversions": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
-      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
-      "license": "BSD-2-Clause",
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/whatwg-url": {
-      "version": "14.2.0",
-      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
-      "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
-      "license": "MIT",
-      "dependencies": {
-        "tr46": "^5.1.0",
-        "webidl-conversions": "^7.0.0"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/xxhashjs": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
-      "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
-      "license": "MIT",
-      "dependencies": {
-        "cuint": "^0.2.2"
-      }
     }
   }
 }

package.json

@@ -25,14 +25,11 @@
     "@aws-sdk/client-s3": "^3.964.0",
     "@aws-sdk/lib-storage": "^3.964.0",
     "@owlboard/backend-data-contracts": "^0.1.9",
-    "mongodb": "^7.0.0",
     "nats": "^2.29.3",
-    "readline": "^1.3.0",
+    "readline": "^1.3.0"
-    "xxhashjs": "^0.2.2"
   },
   "devDependencies": {
     "@types/node": "^25.0.3",
-    "@types/xxhashjs": "^0.2.4",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3"
   }

src/process.ts

@@ -1,6 +1,5 @@
 import { Readable } from 'node:stream';
 import { createInterface } from 'node:readline';
-import XXH from 'xxhashjs';
 import { log } from './logger.js';
 import { DataIngressPisData } from '@owlboard/backend-data-contracts';
 import type { GeneralConfig } from './config.js';
@@ -38,17 +37,17 @@ export async function* processPisStream(cfg: GeneralConfig, inputStream: Readabl
         const record = JSON.parse(line) as InputRecord;
         // log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
 
-        const crsHash = XXH.h64(record.stops.join('|'), SEED);
+        const normalisedStops = record.stops.map(stop => stop.toUpperCase());
-        const tiplocStops = await mapStopsToTiploc(record.stops);
+
-        const tiplocHash = XXH.h64(tiplocStops.join('|'), SEED);
+        const tiplocStops = await mapStopsToTiploc(normalisedStops);
 
         const data: DataIngressPisData.PisObjects = {
             code: record.code,
             toc: TOC.toLowerCase(),
-            crsStops: record.stops,
+            crsStops: normalisedStops,
-            crsHash: crsHash.toString(),
+            crsHash: "",
             tiplocStops: tiplocStops,
-            tiplocHash: tiplocHash.toString(),
+            tiplocHash: "",
         }
         yield data;
     
@@ -64,8 +63,9 @@ async function mapStopsToTiploc(crsStops: string[]): Promise<string[]> {
         // Cache Miss
         try {
             const tiploc = await fetchTiplocFromApi(crs);
-            tiplocCache.set(crs, tiploc);
+            const normalisedTiploc = tiploc.toUpperCase();
-            return tiploc;
+            tiplocCache.set(crs, normalisedTiploc);
+            return normalisedTiploc;
         } catch (err) {
             log('ERROR', `Failed lookup for: ${crs}`, err);
             process.exit(99);