diff --git a/package-lock.json b/package-lock.json index 509a77b..c46004c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,14 +12,11 @@ "@aws-sdk/client-s3": "^3.964.0", "@aws-sdk/lib-storage": "^3.964.0", "@owlboard/backend-data-contracts": "^0.1.9", - "mongodb": "^7.0.0", "nats": "^2.29.3", - "readline": "^1.3.0", - "xxhashjs": "^0.2.2" + "readline": "^1.3.0" }, "devDependencies": { "@types/node": "^25.0.3", - "@types/xxhashjs": "^0.2.4", "tsx": "^4.21.0", "typescript": "^5.9.3" } @@ -1385,15 +1382,6 @@ "node": ">=18" } }, - "node_modules/@mongodb-js/saslprep": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.4.tgz", - "integrity": "sha512-p7X/ytJDIdwUfFL/CLOhKgdfJe1Fa8uw9seJYvdOmnP9JBWGWHW69HkOixXS6Wy9yvGf1MbhcS6lVmrhy4jm2g==", - "license": "MIT", - "dependencies": { - "sparse-bitfield": "^3.0.3" - } - }, "node_modules/@owlboard/backend-data-contracts": { "version": "0.1.9", "resolved": "https://git.fjla.uk/api/packages/OwlBoard/npm/%40owlboard%2Fbackend-data-contracts/-/0.1.9/backend-data-contracts-0.1.9.tgz", @@ -2142,31 +2130,6 @@ "undici-types": "~7.16.0" } }, - "node_modules/@types/webidl-conversions": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", - "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", - "license": "MIT" - }, - "node_modules/@types/whatwg-url": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-13.0.0.tgz", - "integrity": "sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==", - "license": "MIT", - "dependencies": { - "@types/webidl-conversions": "*" - } - }, - "node_modules/@types/xxhashjs": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.4.tgz", - "integrity": "sha512-E2+ZoJY2JjmVPN0iQM5gJvZkk98O2PYXSi6HrciEk3EKF34+mauEk/HgwTeCz+2r8HXHMKpucrwy4qTT12OPaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -2193,15 +2156,6 @@ "integrity": "sha512-OHawaAbjwx6rqICCKgSG0SAnT05bzd7ppyKLVUITZpANBaaMFBAsaNkto3LoQ31tyFP5kNujE8Cdx85G9VzOkw==", "license": "MIT" }, - "node_modules/bson": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/bson/-/bson-7.0.0.tgz", - "integrity": "sha512-Kwc6Wh4lQ5OmkqqKhYGKIuELXl+EPYSCObVE6bWsp1T/cGkOCBN0I8wF/T44BiuhHyNi1mmKVPXk60d41xZ7kw==", - "license": "Apache-2.0", - "engines": { - "node": ">=20.19.0" - } - }, "node_modules/buffer": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", @@ -2212,12 +2166,6 @@ "ieee754": "^1.1.4" } }, - "node_modules/cuint": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", - "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==", - "license": "MIT" - }, "node_modules/esbuild": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", @@ -2341,71 +2289,6 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/memory-pager": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", - "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", - "license": "MIT" - }, - "node_modules/mongodb": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.0.0.tgz", - "integrity": "sha512-vG/A5cQrvGGvZm2mTnCSz1LUcbOPl83hfB6bxULKQ8oFZauyox/2xbZOoGNl+64m8VBrETkdGCDBdOsCr3F3jg==", - "license": "Apache-2.0", - "dependencies": { - "@mongodb-js/saslprep": "^1.3.0", - "bson": "^7.0.0", - "mongodb-connection-string-url": "^7.0.0" - }, - "engines": { - "node": ">=20.19.0" - }, - "peerDependencies": { - "@aws-sdk/credential-providers": "^3.806.0", - "@mongodb-js/zstd": "^7.0.0", - "gcp-metadata": "^7.0.1", - "kerberos": "^7.0.0", - "mongodb-client-encryption": ">=7.0.0 <7.1.0", - "snappy": "^7.3.2", - "socks": "^2.8.6" - }, - "peerDependenciesMeta": { - "@aws-sdk/credential-providers": { - "optional": true - }, - "@mongodb-js/zstd": { - "optional": true - }, - "gcp-metadata": { - "optional": true - }, - "kerberos": { - "optional": true - }, - "mongodb-client-encryption": { - "optional": true - }, - "snappy": { - "optional": true - }, - "socks": { - "optional": true - } - } - }, - "node_modules/mongodb-connection-string-url": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-7.0.0.tgz", - "integrity": "sha512-irhhjRVLE20hbkRl4zpAYLnDMM+zIZnp0IDB9akAFFUZp/3XdOfwwddc7y6cNvF2WCEtfTYRwYbIfYa2kVY0og==", - "license": "Apache-2.0", - "dependencies": { - "@types/whatwg-url": "^13.0.0", - "whatwg-url": "^14.1.0" - }, - "engines": { - "node": ">=20.19.0" - } - }, "node_modules/nats": { "version": "2.29.3", "resolved": "https://registry.npmjs.org/nats/-/nats-2.29.3.tgz", @@ -2430,15 +2313,6 @@ "node": ">=10.0.0" } }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -2489,15 +2363,6 @@ ], "license": "MIT" }, - "node_modules/sparse-bitfield": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", - "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", - "license": "MIT", - "dependencies": { - "memory-pager": "^1.0.2" - } - }, "node_modules/stream-browserify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", @@ -2529,18 +2394,6 @@ ], "license": "MIT" }, - "node_modules/tr46": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", - "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", - "license": "MIT", - "dependencies": { - "punycode": "^2.3.1" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -2599,37 +2452,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" - }, - "node_modules/webidl-conversions": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", - "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - } - }, - "node_modules/whatwg-url": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", - "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", - "license": "MIT", - "dependencies": { - "tr46": "^5.1.0", - "webidl-conversions": "^7.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/xxhashjs": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", - "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", - "license": "MIT", - "dependencies": { - "cuint": "^0.2.2" - } } } } diff --git a/package.json b/package.json index f383a3e..162406c 100644 --- a/package.json +++ b/package.json @@ -25,14 +25,11 @@ "@aws-sdk/client-s3": "^3.964.0", "@aws-sdk/lib-storage": "^3.964.0", "@owlboard/backend-data-contracts": "^0.1.9", - "mongodb": "^7.0.0", "nats": "^2.29.3", - "readline": "^1.3.0", - "xxhashjs": "^0.2.2" + "readline": "^1.3.0" }, "devDependencies": { "@types/node": "^25.0.3", - "@types/xxhashjs": "^0.2.4", "tsx": "^4.21.0", "typescript": "^5.9.3" } diff --git a/src/process.ts b/src/process.ts index 53c74b7..9ec57ac 100644 --- a/src/process.ts +++ b/src/process.ts @@ -1,6 +1,5 @@ import { Readable } from 'node:stream'; import { createInterface } from 'node:readline'; -import XXH from 'xxhashjs'; import { log } from './logger.js'; import { DataIngressPisData } from '@owlboard/backend-data-contracts'; import type { GeneralConfig } from './config.js'; @@ -38,17 +37,17 @@ export async function* processPisStream(cfg: GeneralConfig, inputStream: Readabl const record = JSON.parse(line) as InputRecord; // log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`) - const crsHash = XXH.h64(record.stops.join('|'), SEED); - const tiplocStops = await mapStopsToTiploc(record.stops); - const tiplocHash = XXH.h64(tiplocStops.join('|'), SEED); + const normalisedStops = record.stops.map(stop => stop.toUpperCase()); + + const tiplocStops = await mapStopsToTiploc(normalisedStops); const data: DataIngressPisData.PisObjects = { code: record.code, toc: TOC.toLowerCase(), - crsStops: record.stops, - crsHash: crsHash.toString(), + crsStops: normalisedStops, + crsHash: "", tiplocStops: tiplocStops, - tiplocHash: tiplocHash.toString(), + tiplocHash: "", } yield data; @@ -64,8 +63,9 @@ async function mapStopsToTiploc(crsStops: string[]): Promise { // Cache Miss try { const tiploc = await fetchTiplocFromApi(crs); - tiplocCache.set(crs, tiploc); - return tiploc; + const normalisedTiploc = tiploc.toUpperCase(); + tiplocCache.set(crs, normalisedTiploc); + return normalisedTiploc; } catch (err) { log('ERROR', `Failed lookup for: ${crs}`, err); process.exit(99);