Add fetch and stream to temporary file

src/process.ts

@@ -0,0 +1,78 @@
+import { Readable } from 'node:stream';
+import { createInterface } from 'node:readline';
+import XXH from 'xxhashjs';
+import { log } from './logger.js';
+
+const BASE_URL = process.env.BASEURL || 'https://owlboard.info'
+const tiplocCache = new Map<string, string>();
+const SEED = 0;
+interface InputRecord {
+    code: string;
+    stops: string[];
+}
+
+export async function* processPisStream(inputStream: Readable) {
+    const TOC = process.env.TOC;
+    if (!TOC) {
+        log('ERROR', "TOC not set: Exit code 19");
+        process.exit(19);
+    }
+    log('DEBUG', `TOC set to: ${TOC}`)
+
+    const rl = createInterface({
+        input: inputStream,
+        terminal: false,
+    });
+
+    for await (const line of rl) {
+        if (!line.trim()) continue;
+
+        const record = JSON.parse(line) as InputRecord;
+        log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
+
+        const crsHash = XXH.h64(record.stops.join('|'), SEED);
+        const tiplocStops = await mapStopsToTiploc(record.stops);
+        const tiplocHash = XXH.h64(tiplocStops.join('|'), SEED);
+        yield {
+            code: record.code,
+            toc: TOC.toLowerCase(),
+            crsStops: record.stops,
+            crsHash: crsHash.toString(10),
+            tiplocStops: tiplocStops,
+            tiplocHash: tiplocHash.toString(10),
+        }
+    
+    }
+}
+
+async function mapStopsToTiploc(crsStops: string[]): Promise<string[]> {
+    return Promise.all(crsStops.map(async (crs) => {
+        if (tiplocCache.has(crs)) {
+            return tiplocCache.get(crs)!;
+        }
+
+        // Cache Miss
+        try {
+            const tiploc = await fetchTiplocFromApi(crs);
+            tiplocCache.set(crs, tiploc);
+            return tiploc;
+        } catch (err) {
+            log('ERROR', `Failed lookup for: ${crs}`, err);
+            process.exit(99);
+        }
+    }))
+}
+
+async function fetchTiplocFromApi(crs: string): Promise<string> {
+    const apiUrl = `${BASE_URL}/api/v2/ref/locationCode/crs/${crs}`;
+    const response = await fetch(apiUrl, {
+        headers: { 'Accept': 'application/json' },
+    });
+
+    if (!response.ok) {
+        throw new Error(`API Error: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data[0].TIPLOC;
+}