Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a2d6ee7400 | |||
| 7903344eea |
1430
package-lock.json
generated
1430
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -25,14 +25,11 @@
|
|||||||
"@aws-sdk/client-s3": "^3.964.0",
|
"@aws-sdk/client-s3": "^3.964.0",
|
||||||
"@aws-sdk/lib-storage": "^3.964.0",
|
"@aws-sdk/lib-storage": "^3.964.0",
|
||||||
"@owlboard/backend-data-contracts": "^0.1.9",
|
"@owlboard/backend-data-contracts": "^0.1.9",
|
||||||
"mongodb": "^7.0.0",
|
|
||||||
"nats": "^2.29.3",
|
"nats": "^2.29.3",
|
||||||
"readline": "^1.3.0",
|
"readline": "^1.3.0"
|
||||||
"xxhashjs": "^0.2.2"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^25.0.3",
|
"@types/node": "^25.0.3",
|
||||||
"@types/xxhashjs": "^0.2.4",
|
|
||||||
"tsx": "^4.21.0",
|
"tsx": "^4.21.0",
|
||||||
"typescript": "^5.9.3"
|
"typescript": "^5.9.3"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
import { Readable } from 'node:stream';
|
import { Readable } from 'node:stream';
|
||||||
import { createInterface } from 'node:readline';
|
import { createInterface } from 'node:readline';
|
||||||
import XXH from 'xxhashjs';
|
|
||||||
import { log } from './logger.js';
|
import { log } from './logger.js';
|
||||||
import { DataIngressPisData } from '@owlboard/backend-data-contracts';
|
import { DataIngressPisData } from '@owlboard/backend-data-contracts';
|
||||||
import type { GeneralConfig } from './config.js';
|
import type { GeneralConfig } from './config.js';
|
||||||
@@ -38,17 +37,17 @@ export async function* processPisStream(cfg: GeneralConfig, inputStream: Readabl
|
|||||||
const record = JSON.parse(line) as InputRecord;
|
const record = JSON.parse(line) as InputRecord;
|
||||||
// log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
|
// log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
|
||||||
|
|
||||||
const crsHash = XXH.h64(record.stops.join('|'), SEED);
|
const normalisedStops = record.stops.map(stop => stop.toUpperCase());
|
||||||
const tiplocStops = await mapStopsToTiploc(record.stops);
|
|
||||||
const tiplocHash = XXH.h64(tiplocStops.join('|'), SEED);
|
const tiplocStops = await mapStopsToTiploc(normalisedStops);
|
||||||
|
|
||||||
const data: DataIngressPisData.PisObjects = {
|
const data: DataIngressPisData.PisObjects = {
|
||||||
code: record.code,
|
code: record.code,
|
||||||
toc: TOC.toLowerCase(),
|
toc: TOC.toLowerCase(),
|
||||||
crsStops: record.stops,
|
crsStops: normalisedStops,
|
||||||
crsHash: crsHash.toString(),
|
crsHash: "",
|
||||||
tiplocStops: tiplocStops,
|
tiplocStops: tiplocStops,
|
||||||
tiplocHash: tiplocHash.toString(),
|
tiplocHash: "",
|
||||||
}
|
}
|
||||||
yield data;
|
yield data;
|
||||||
|
|
||||||
@@ -64,8 +63,9 @@ async function mapStopsToTiploc(crsStops: string[]): Promise<string[]> {
|
|||||||
// Cache Miss
|
// Cache Miss
|
||||||
try {
|
try {
|
||||||
const tiploc = await fetchTiplocFromApi(crs);
|
const tiploc = await fetchTiplocFromApi(crs);
|
||||||
tiplocCache.set(crs, tiploc);
|
const normalisedTiploc = tiploc.toUpperCase();
|
||||||
return tiploc;
|
tiplocCache.set(crs, normalisedTiploc);
|
||||||
|
return normalisedTiploc;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log('ERROR', `Failed lookup for: ${crs}`, err);
|
log('ERROR', `Failed lookup for: ${crs}`, err);
|
||||||
process.exit(99);
|
process.exit(99);
|
||||||
|
|||||||
Reference in New Issue
Block a user