OwlBoard/pis-data-ingress
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fetch and stream to temporary file

Browse Source
This commit is contained in:
Fred Boniface
2026-01-06 20:58:12 +00:00
parent 49b46f5e9c
commit 03ac6431ff
7 changed files with 728 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

586
package-lock.json generated
View File

@@ -9,13 +9,466 @@
"version": "0.0.1",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@owlboard/backend-data-contracts": "^0.0.1"
"@bufbuild/protobuf": "^2.10.2",
"@owlboard/backend-data-contracts": "^0.0.1",
"readline": "^1.3.0",
"xxhashjs": "^0.2.2"
},
"devDependencies": {
"@types/node": "^25.0.3",
"@types/xxhashjs": "^0.2.4",
"tsx": "^4.21.0",
"typescript": "^5.9.3"
}
},
"node_modules/@bufbuild/protobuf": {
"version": "2.10.2",
"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.10.2.tgz",
"integrity": "sha512-uFsRXwIGyu+r6AMdz+XijIIZJYpoWeYzILt5yZ2d3mCjQrWUTVpVD9WL/jZAbvp+Ed04rOhrsk7FiTcEDseB5A==",
"license": "(Apache-2.0 AND BSD-3-Clause)"
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz",
"integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"aix"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz",
"integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz",
"integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz",
"integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz",
"integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz",
"integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz",
"integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz",
"integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz",
"integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz",
"integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ia32": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz",
"integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz",
"integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==",
"cpu": [
"loong64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-mips64el": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz",
"integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==",
"cpu": [
"mips64el"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ppc64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz",
"integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-riscv64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz",
"integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==",
"cpu": [
"riscv64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-s390x": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz",
"integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==",
"cpu": [
"s390x"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz",
"integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz",
"integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz",
"integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz",
"integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz",
"integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openharmony-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz",
"integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openharmony"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/sunos-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz",
"integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-arm64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz",
"integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-ia32": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz",
"integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-x64": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz",
"integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@owlboard/backend-data-contracts": {
"version": "0.0.1",
"resolved": "https://git.fjla.uk/api/packages/OwlBoard/npm/%40owlboard%2Fbackend-data-contracts/-/0.0.1/backend-data-contracts-0.0.1.tgz",
@@ -32,6 +485,128 @@
"undici-types": "~7.16.0"
}
},
"node_modules/@types/xxhashjs": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/@types/xxhashjs/-/xxhashjs-0.2.4.tgz",
"integrity": "sha512-E2+ZoJY2JjmVPN0iQM5gJvZkk98O2PYXSi6HrciEk3EKF34+mauEk/HgwTeCz+2r8HXHMKpucrwy4qTT12OPaQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/cuint": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
"integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==",
"license": "MIT"
},
"node_modules/esbuild": {
"version": "0.27.2",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz",
"integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"@esbuild/aix-ppc64": "0.27.2",
"@esbuild/android-arm": "0.27.2",
"@esbuild/android-arm64": "0.27.2",
"@esbuild/android-x64": "0.27.2",
"@esbuild/darwin-arm64": "0.27.2",
"@esbuild/darwin-x64": "0.27.2",
"@esbuild/freebsd-arm64": "0.27.2",
"@esbuild/freebsd-x64": "0.27.2",
"@esbuild/linux-arm": "0.27.2",
"@esbuild/linux-arm64": "0.27.2",
"@esbuild/linux-ia32": "0.27.2",
"@esbuild/linux-loong64": "0.27.2",
"@esbuild/linux-mips64el": "0.27.2",
"@esbuild/linux-ppc64": "0.27.2",
"@esbuild/linux-riscv64": "0.27.2",
"@esbuild/linux-s390x": "0.27.2",
"@esbuild/linux-x64": "0.27.2",
"@esbuild/netbsd-arm64": "0.27.2",
"@esbuild/netbsd-x64": "0.27.2",
"@esbuild/openbsd-arm64": "0.27.2",
"@esbuild/openbsd-x64": "0.27.2",
"@esbuild/openharmony-arm64": "0.27.2",
"@esbuild/sunos-x64": "0.27.2",
"@esbuild/win32-arm64": "0.27.2",
"@esbuild/win32-ia32": "0.27.2",
"@esbuild/win32-x64": "0.27.2"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/get-tsconfig": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
"integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"resolve-pkg-maps": "^1.0.0"
},
"funding": {
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
}
},
"node_modules/readline": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz",
"integrity": "sha512-k2d6ACCkiNYz222Fs/iNze30rRJ1iIicW7JuX/7/cozvih6YCkFZH+J6mAFDVgv0dRBaAyr4jDqC95R2y4IADg==",
"license": "BSD"
},
"node_modules/resolve-pkg-maps": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
"dev": true,
"license": "MIT",
"funding": {
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
}
},
"node_modules/tsx": {
"version": "4.21.0",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
"integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
"dev": true,
"license": "MIT",
"dependencies": {
"esbuild": "~0.27.0",
"get-tsconfig": "^4.7.5"
},
"bin": {
"tsx": "dist/cli.mjs"
},
"engines": {
"node": ">=18.0.0"
},
"optionalDependencies": {
"fsevents": "~2.3.3"
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
@@ -52,6 +627,15 @@
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
},
"node_modules/xxhashjs": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
"integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
"license": "MIT",
"dependencies": {
"cuint": "^0.2.2"
}
}
}
}

8
package.json
View File

@@ -3,6 +3,7 @@
"version": "0.0.1",
"description": "PIS Data Ingress for OwlBoard",
"main": "index.js",
"type": "module",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
@@ -18,10 +19,15 @@
"author": "Frederick Boniface",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@owlboard/backend-data-contracts": "^0.0.1"
"@bufbuild/protobuf": "^2.10.2",
"@owlboard/backend-data-contracts": "^0.0.1",
"readline": "^1.3.0",
"xxhashjs": "^0.2.2"
},
"devDependencies": {
"@types/node": "^25.0.3",
"@types/xxhashjs": "^0.2.4",
"tsx": "^4.21.0",
"typescript": "^5.9.3"
}
}

47
src/index.ts
View File

@@ -1 +1,46 @@
import { log } from './logger'
import { log } from './logger.js'
import { Readable } from 'node:stream'
import { createWriteStream } from 'node:fs'
import { finished } from 'node:stream/promises'
import { getLatestPackageName, getRequestStream } from './sources/gitea.js'
import { processPisStream } from './process.js'
async function main() {
try {
const packageInfo = await getLatestPackageName();
log('INFO', `Latest PIS Package: ${packageInfo.name}`);
if (!packageInfo.assets[0]?.browser_download_url) {
log('ERROR', `No attachments found for release ${packageInfo.name}`);
process.exit(9);
}
const inputStream: Readable = await getRequestStream(packageInfo.assets[0].browser_download_url);
const outputPath = './review_output.ndjson';
log('DEBUG', `Processing stream to: ${outputPath}`);
await writeToTemporaryFile(processPisStream(inputStream), outputPath);
log('DEBUG', 'Done');
} catch (err) {
log('ERROR', 'Fatal error in pipeline: ', err);
process.exit(7);
}
}
async function writeToTemporaryFile(generator: AsyncGenerator<any>, path: string) {
const writer = createWriteStream(path);
for await (const record of generator) {
const line = JSON.stringify(record) + '\n';
if (!writer.write(line)) {
await new Promise((resolve) => writer.once('drain', resolve));
}
}
writer.end();
return finished(writer);
}
main();

2
src/logger.ts
View File

@@ -15,7 +15,7 @@ const levelWeight: Record<LogLevel, number> = {
const RESET = '\x1b[0m';
const CURRENT_LOG_LEVEL = (process.env.LOG_LEVEL as LogLevel) || 'INFO';
const CURRENT_LOG_LEVEL = (process.env.LOG_LEVEL as LogLevel) || 'DEBUG';
const MIN_WEIGHT = levelWeight[CURRENT_LOG_LEVEL] ?? 1;
export function log(level: LogLevel, msg: string, ...args: any[]): void {

78
src/process.ts Normal file
View File

@@ -0,0 +1,78 @@
import { Readable } from 'node:stream';
import { createInterface } from 'node:readline';
import XXH from 'xxhashjs';
import { log } from './logger.js';
const BASE_URL = process.env.BASEURL || 'https://owlboard.info'
const tiplocCache = new Map<string, string>();
const SEED = 0;
interface InputRecord {
code: string;
stops: string[];
}
export async function* processPisStream(inputStream: Readable) {
const TOC = process.env.TOC;
if (!TOC) {
log('ERROR', "TOC not set: Exit code 19");
process.exit(19);
}
log('DEBUG', `TOC set to: ${TOC}`)
const rl = createInterface({
input: inputStream,
terminal: false,
});
for await (const line of rl) {
if (!line.trim()) continue;
const record = JSON.parse(line) as InputRecord;
log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
const crsHash = XXH.h64(record.stops.join('|'), SEED);
const tiplocStops = await mapStopsToTiploc(record.stops);
const tiplocHash = XXH.h64(tiplocStops.join('|'), SEED);
yield {
code: record.code,
toc: TOC.toLowerCase(),
crsStops: record.stops,
crsHash: crsHash.toString(10),
tiplocStops: tiplocStops,
tiplocHash: tiplocHash.toString(10),
}
}
}
async function mapStopsToTiploc(crsStops: string[]): Promise<string[]> {
return Promise.all(crsStops.map(async (crs) => {
if (tiplocCache.has(crs)) {
return tiplocCache.get(crs)!;
}
// Cache Miss
try {
const tiploc = await fetchTiplocFromApi(crs);
tiplocCache.set(crs, tiploc);
return tiploc;
} catch (err) {
log('ERROR', `Failed lookup for: ${crs}`, err);
process.exit(99);
}
}))
}
async function fetchTiplocFromApi(crs: string): Promise<string> {
const apiUrl = `${BASE_URL}/api/v2/ref/locationCode/crs/${crs}`;
const response = await fetch(apiUrl, {
headers: { 'Accept': 'application/json' },
});
if (!response.ok) {
throw new Error(`API Error: ${response.status}`);
}
const data = await response.json();
return data[0].TIPLOC;
}

18
src/sources/gitea.ts
View File

@@ -1,11 +1,11 @@
import { Readable } from 'node:stream';
import { log } from '../logger';
import { log } from '../logger.js';
const GITEA_TOKEN = process.env.GITEA_TOKEN as string || "";
const GITEA_SERVER = process.env.GITEA_SERVER as string || "";
const GITEA_OWNER = process.env.GITEA_OWNER as string || "";
const GITEA_REPO = process.env.GITEA_REPO as string || "";
const GITEA_PKG_URL = `https://${GITEA_SERVER}/${GITEA_OWNER}/${GITEA_REPO}`
const GITEA_PKG_URL = `https://${GITEA_SERVER}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}`
const REQUIRED_ENV_VARS = [
"GITEA_SERVER",
@@ -36,7 +36,8 @@ if (missing) {
interface GiteaPackageAsset {
name: string;
id: string;
id: number;
browser_download_url: string;
}
export interface GiteaPackageResponse {
id: number;
@@ -47,9 +48,12 @@ export interface GiteaPackageResponse {
}
export async function getLatestPackageName() :Promise<GiteaPackageResponse> {
const get = await sendRequest<GiteaPackageResponse>(`${GITEA_PKG_URL}/releases/latest`);
const url = `${GITEA_PKG_URL}/releases/latest`
log("DEBUG", `Sending request to ${url}`)
const get = await sendRequest<GiteaPackageResponse>(url);
if (get.name) {
log("DEBUG", `Latest package name: ${get.name}`)
return get
} else {
log("ERROR", "Invalid Resposne from Gitea Server");
@@ -58,11 +62,7 @@ export async function getLatestPackageName() :Promise<GiteaPackageResponse> {
}
}
export async function getPackageAttachment(release_id: number, attachment_id: number) :Promise<any> {
return getRequestStream(`${GITEA_PKG_URL}/releases/${release_id}/assets/${attachment_id}`);
}
async function getRequestStream(url: string) :Promise<Readable> {
export async function getRequestStream(url: string) :Promise<Readable> {
const options: RequestInit = {
method: "GET",
headers: {

4
tsconfig.json
View File

@@ -2,8 +2,8 @@
// Visit https://aka.ms/tsconfig to read more about this file
"compilerOptions": {
// File Layout
// "rootDir": "./src",
// "outDir": "./dist",
"rootDir": "./src",
"outDir": "./dist",
// Environment Settings
// See also https://aka.ms/tsconfig/module