Ensure S3 configured to create bucket if not exists

Switch Dockerfile to use absolute paths

.dockerignore

@@ -0,0 +1,138 @@
+# ---> Node
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# vitepress build output
+**/.vitepress/dist
+
+# vitepress cache directory
+**/.vitepress/cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+

.gitea/workflows/build-push.yaml

@@ -0,0 +1,39 @@
+name: Build and Push container image
+run-name: ${{ gitea.actor }} is building and pushing
+
+on:
+  create:
+    tags: "*"
+
+env:
+  GITEA_DOMAIN: git.fjla.uk
+  GITEA_REGISTRY_USER: owlbot
+  RESULT_IMAGE_NAME: owlboard/data-ingress-pis
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    container:
+      image: catthehacker/ubuntu:act-latest
+      options: --privileged
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.GITEA_DOMAIN }}
+          username: ${{ env.GITEA_REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+      - name: Build and Push image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ env.GITEA_DOMAIN }}/${{ env.RESULT_IMAGE_NAME }}:${{ gitea.ref_name }}
+            ${{ env.GITEA_DOMAIN }}/${{ env.RESULT_IMAGE_NAME }}:latest

Dockerfile

@@ -0,0 +1,27 @@
+FROM node:25-slim AS builder
+
+WORKDIR /app
+
+COPY .npmrc ./
+COPY package*.json ./
+RUN npm ci
+
+COPY . .
+
+RUN npm run build
+
+
+FROM node:25-slim AS runner
+
+ENV NODE_ENV=production
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci --omit=dev
+
+COPY --from=builder /app/dist /app/dist
+
+USER node
+
+CMD ["node", "dist/index.js"]

package-lock.json

@@ -11,7 +11,7 @@
       "dependencies": {
         "@aws-sdk/client-s3": "^3.964.0",
         "@aws-sdk/lib-storage": "^3.964.0",
-        "@owlboard/backend-data-contracts": "^0.1.0",
+        "@owlboard/backend-data-contracts": "^0.1.9",
         "mongodb": "^7.0.0",
         "nats": "^2.29.3",
         "readline": "^1.3.0",
@@ -1395,9 +1395,9 @@
       }
     },
     "node_modules/@owlboard/backend-data-contracts": {
-      "version": "0.1.0",
-      "resolved": "https://git.fjla.uk/api/packages/OwlBoard/npm/%40owlboard%2Fbackend-data-contracts/-/0.1.0/backend-data-contracts-0.1.0.tgz",
-      "integrity": "sha512-6cJg7l7i8+iogU5nzvXct1+MQLmqgwcC3I7aS8MAHiDE3ymnLBhpaFKNfz6zkG39KQbA0XGhtoQD1dlKZzLXfw==",
+      "version": "0.1.9",
+      "resolved": "https://git.fjla.uk/api/packages/OwlBoard/npm/%40owlboard%2Fbackend-data-contracts/-/0.1.9/backend-data-contracts-0.1.9.tgz",
+      "integrity": "sha512-sKIogfclUKSwIvcPFQmJfpVMGLR1XbqMhyL1qNlKFWqWqATCrgKdboArwj5AfTjEC8hDUp8x0ffaRL+hQphXoQ==",
       "license": "ISC"
     },
     "node_modules/@smithy/abort-controller": {

package.json

@@ -5,6 +5,9 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
+    "prebuild": "rm -rf ./dist",
+    "build": "tsc",
+    "start": "node dist/index.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "repository": {
@@ -21,7 +24,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.964.0",
     "@aws-sdk/lib-storage": "^3.964.0",
-    "@owlboard/backend-data-contracts": "^0.1.0",
+    "@owlboard/backend-data-contracts": "^0.1.9",
     "mongodb": "^7.0.0",
     "nats": "^2.29.3",
     "readline": "^1.3.0",

src/config.ts

@@ -0,0 +1,79 @@
+import { readFileSync, existsSync } from "node:fs"
+import { join } from "node:path"
+import { log } from "./logger.js";
+
+interface Config {
+    Mq: Mq,
+    S3: S3,
+    Mongo: Mongo,
+    General: GeneralConfig,
+}
+
+export interface Mq {
+    MQ_USER: string;
+    MQ_PASS: string;
+    MQ_URL: string;
+    MQ_TOPIC: string;
+}
+
+export interface S3 {
+    S3_ENDPOINT: string;
+    S3_BUCKET: string;
+    S3_ACCESS_KEY: string;
+    S3_SECRET_KEY: string;
+    S3_REGION: string;
+}
+
+export interface Mongo {
+    MONGO_URI: string;
+    MONGO_DB: string;
+    MONGO_USER: string;
+    MONGO_PASS: string;
+}
+
+export interface GeneralConfig {
+    TOC: string;
+}
+
+export function ConfigLoader(): Config {
+    const cfg: Config = {
+        Mq: {
+            MQ_URL: getConfig("MQ_URL"),
+            MQ_TOPIC: getConfig("MQ_TOPIC"),
+            MQ_USER: getConfig("MQ_USER"),
+            MQ_PASS: getConfig("MQ_PASS"),
+        },
+        S3: {
+            S3_ENDPOINT: getConfig("S3_ENDPOINT"),
+            S3_BUCKET: getConfig("S3_BUCKET"),
+            S3_ACCESS_KEY: getConfig("S3_ACCESS_KEY"),
+            S3_SECRET_KEY: getConfig("S3_SECRET_KEY"),
+            S3_REGION: getConfig("S3_REGION"),
+        },
+        Mongo: {
+            MONGO_URI: getConfig("MONGO_URI"),
+            MONGO_DB: getConfig("MONGO_DB"),
+            MONGO_USER: getConfig("MONGO_USER"),
+            MONGO_PASS: getConfig("MONGO_PASS"),
+        },
+        General: {
+            TOC: getConfig("TOC"),
+        }
+    };
+
+    return cfg
+}
+
+function getConfig(key: string): string {
+    const filePath = join("/etc/secrets", key);
+    if (existsSync(filePath)) {
+        try {
+            return readFileSync(filePath, "utf8").trim();
+        } catch (err) {
+            log("ERROR", `Error reading secret file at ${filePath}: ${err}`);
+        }
+    }
+
+    return process.env[key] || ""
+}
+

src/database.ts

@@ -1,62 +0,0 @@
-// Functions to check the current applied version
-// updating will be handled by the processing service
-
-import { MongoClient } from "mongodb";
-import { log } from "./logger";
-
-const uri = process.env.MONGO_URI || "";
-const db = process.env.MONGO_DB || "";
-const user = process.env.MONGO_USER || "";
-const pass = process.env.MONGO_PASS || "";
-const collection = "data_ingress_meta";
-
-if(!uri || !db || !user || !pass) {
-    log('ERROR', "Missing MONGO Configuration - EXIT CODE: 35");
-    process.exit(35);
-} else {
-    log("DEBUG", `MongoDB Connection`, {
-        uri: uri,
-        db: db,
-        collection: collection,
-        user: user,
-        pass: "****",
-    });
-};
-
-const CONNECTION_URI = `mongodb://${encodeURIComponent(user)}:${encodeURIComponent(pass)}@${uri}`;
-
-let mongoClient: MongoClient | null = null;
-
-async function getMongoClient() {
-    if (mongoClient) return mongoClient;
-
-    mongoClient = new MongoClient(CONNECTION_URI);
-    await mongoClient.connect();
-    return mongoClient;
-}
-
-export async function isPackageProcessed(serviceName: string, packageName: string): Promise<boolean> {
-    try {
-        const client = await getMongoClient();
-        const database = client.db(db);
-        const coll = database.collection(collection);
-
-        const result = await coll.findOne({ service_name: serviceName });
-
-        if (!result) {
-            log('INFO', `No metadata found for ${serviceName}.  Fetching PIS Data`);
-            return false;
-        }
-
-        if (result.latest_package === packageName) {
-            log('INFO', 'No update needed');
-            return true;
-        }
-
-        log('INFO', `Version mismatch.  DB: ${result.latest_package}, Current: ${packageName}.  Update required`)
-        return false;
-    } catch (err) {
-        log('ERROR', 'Failed to check Mongo for version state:', err);
-        process.exit(1);
-    }
-}

src/index.ts

@@ -4,7 +4,8 @@ import { processAndStore } from './sss.js'
 
 import { getLatestPackageName, getRequestStream } from './sources/gitea.js'
 import { processPisStream } from './process.js'
-import { isPackageProcessed } from './database.js'
+import { ConfigLoader } from './config.js'
+import { sendFileUpdateMessage } from './nats.js'
 
 async function main() {
     const SERVICE_NAME = process.env.SERVICE_NAME;
@@ -13,6 +14,8 @@ async function main() {
         process.exit(1);
     }
 
+    const config = ConfigLoader();
+
     try {
         const packageInfo = await getLatestPackageName();
         log('INFO', `Latest PIS Package: ${packageInfo.name}`);
@@ -22,20 +25,24 @@ async function main() {
             process.exit(9);
         }
 
-        if (await isPackageProcessed(SERVICE_NAME, packageInfo.name)) {
-            log('INFO', `Database matches latest release.  Exiting`);
-            process.exit(0);
-        }
+        // Check if package already processed HERE...
+        log('WARN', 'Version check disabled, downloading data from source');
+        // exit(0) if done, else continue
 
         const inputStream: Readable = await getRequestStream(packageInfo.assets[0].browser_download_url);
-        const objectGenerator = processPisStream(inputStream);
+        const objectGenerator = processPisStream(config.General, inputStream);
 
         const filename = `${packageInfo.name.replace(/\s+/g, '_')}_pis_data_ndjson`;
 
         log('DEBUG', `Processing stream to: ${filename}`);
 
-        await processAndStore(objectGenerator, filename);
+        await processAndStore(config.S3, objectGenerator, filename);
         log('DEBUG', 'Done');
+
+        log('DEBUG', "Sending message to NATS");
+        sendFileUpdateMessage(config.Mq, filename, packageInfo.name, SERVICE_NAME);
+        log('DEBUG', "Done, exiting")
+        return
     } catch (err) {
         log('ERROR', 'Fatal error in pipeline: ', err);
         process.exit(7);

src/nats.ts

@@ -1,22 +1,23 @@
 import { connect, JSONCodec } from "nats";
 import type { ConnectionOptions, NatsConnection, Payload } from "nats";
-import { log } from "./logger";
+import { log } from "./logger.js";
+import { hostname } from "node:os";
+import type { MQFileUpdate } from "@owlboard/backend-data-contracts/dist/data-ingress_mq-file-update";
+import type { Mq } from "./config.js";
 
 const jc = JSONCodec();
 
-async function getNatsConnection(): Promise<NatsConnection> {
-    const serverUrl = process.env.MQ_URL || "nats://localhost:4222";
-
+async function getNatsConnection(cfg: Mq): Promise<NatsConnection> {
     const options: ConnectionOptions = {
-        servers: serverUrl,
-        name: `${process.env.HOSTNAME}` || 'local',
+        servers: cfg.MQ_URL,
+        name: hostname(),
         reconnect: true,
         maxReconnectAttempts: -1,
     };
 
-    if (process.env.MQ_USER && process.env.MQ_PASS) {
-        options.user = process.env.MQ_USER;
-        options.pass = process.env.MQ_PASS;
+    if (cfg.MQ_USER && cfg.MQ_PASS) {
+        options.user = cfg.MQ_USER;
+        options.pass = cfg.MQ_PASS;
         log("INFO", "NATS: Using username/password authentication");
     } else {
         log("INFO", "NATS: Connecting without authentication");
@@ -24,3 +25,36 @@ async function getNatsConnection(): Promise<NatsConnection> {
 
     return await connect(options)
 }
+
+// Send Message Function here to send the message to NATS
+export async function sendFileUpdateMessage(cfg: Mq, path: string, version: string, serviceName: string): Promise<boolean> {
+    const serviceId: string = hostname();
+    const message: MQFileUpdate = {
+        service_name: serviceName,
+        service_id: serviceId,
+        sent_timestamp: Math.floor(Date.now() / 1000),
+        data_type: "file",
+        data_kind: "pis",
+        payload: {
+            version: version,
+            filepath: path,
+        }
+    };
+    
+    let nats: NatsConnection | undefined;
+
+    try {
+        const nats: NatsConnection = await getNatsConnection(cfg);
+
+        const subject = `ingress.file.${message.data_kind}`;
+
+        nats.publish(subject, jc.encode(message));
+
+        await nats.drain();
+        return true
+    } catch (err) {
+        log("ERROR", `NATS: Failed to send message: ${err}`);
+        if (nats) {nats.close()}
+        return false;
+    }
+}

src/process.ts

@@ -3,6 +3,7 @@ import { createInterface } from 'node:readline';
 import XXH from 'xxhashjs';
 import { log } from './logger.js';
 import { DataIngressPisData } from '@owlboard/backend-data-contracts';
+import type { GeneralConfig } from './config.js';
 
 const BASE_URL = process.env.BASEURL || 'https://owlboard.info'
 
@@ -18,8 +19,8 @@ interface InputRecord {
     stops: string[];
 }
 
-export async function* processPisStream(inputStream: Readable) {
-    const TOC = process.env.TOC;
+export async function* processPisStream(cfg: GeneralConfig, inputStream: Readable) {
+    const TOC = cfg.TOC;
     if (!TOC) {
         log('ERROR', "TOC not set: Exit code 19");
         process.exit(19);
@@ -35,7 +36,7 @@ export async function* processPisStream(inputStream: Readable) {
         if (!line.trim()) continue;
 
         const record = JSON.parse(line) as InputRecord;
-        log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
+        // log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
 
         const crsHash = XXH.h64(record.stops.join('|'), SEED);
         const tiplocStops = await mapStopsToTiploc(record.stops);

src/sss.ts

@@ -1,11 +1,13 @@
-import { S3Client } from "@aws-sdk/client-s3";
+import { S3Client, CreateBucketCommand, HeadBucketCommand, S3ServiceException } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { createWriteStream } from "node:fs";
 import { Readable } from "node:stream";
 import { DataIngressPisData } from "@owlboard/backend-data-contracts";
 import { log } from "./logger.js";
+import type { S3 } from "./config.js";
 
 export async function processAndStore(
+    cfg: S3,
     generator: AsyncGenerator<DataIngressPisData.PisObjects>,
     filename: string
 ) {
@@ -15,29 +17,45 @@ export async function processAndStore(
         }
     })());
 
-    const useS3 = process.env.S3_ENDPOINT && process.env.S3_BUCKET;
+    const useS3 = cfg.S3_ENDPOINT && cfg.S3_BUCKET;
 
     if (useS3) {
-        if (!process.env.S3_ENDPOINT || process.env.S3_BUCKET || !process.env.S3_REGION || !process.env.S3_ACCESS_KEY || !process.env.S3_SECRET_KEY) {
+        if (!cfg.S3_ENDPOINT || !cfg.S3_BUCKET || !cfg.S3_REGION || !cfg.S3_ACCESS_KEY || !cfg.S3_SECRET_KEY) {
             log("DEBUG", "Missing required variables for S3 Support - EXIT CODE 24");
             process.exit(24);
         }
-        log('INFO', `Streaming to S3: ${process.env.S3_BUCKET}/${filename}`);
+
+        log("DEBUG", "Opening connection to S3 Server");
 
         const client = new S3Client({
-            endpoint: process.env.S3_ENDPOINT,
-            region: process.env.S3_REGION,
+            endpoint: cfg.S3_ENDPOINT,
+            region: cfg.S3_REGION,
             credentials: {
-                accessKeyId: process.env.S3_ACCESS_KEY!,
-                secretAccessKey: process.env.S3_SECRET_KEY,
+                accessKeyId: cfg.S3_ACCESS_KEY!,
+                secretAccessKey: cfg.S3_SECRET_KEY,
             },
             forcePathStyle: true,
         });
 
+        // Check bucket exists, and create if needed
+        try {
+            await client.send(new HeadBucketCommand({ Bucket: cfg.S3_BUCKET }));
+        } catch (err) {
+            if (err instanceof S3ServiceException && err.$metadata.httpStatusCode === 404) {
+                log('INFO', `Bucket ${cfg.S3_BUCKET} does not exist, creating...`);
+                await client.send(new CreateBucketCommand({ Bucket: cfg.S3_BUCKET }));
+            } else {
+                log(`ERROR`, `Failed to verify bucket: ${cfg.S3_BUCKET}`);
+                process.exit(21);
+            }
+        }
+
+        log('INFO', `Streaming to S3: ${cfg.S3_BUCKET}/${filename}`);
+
         const upload = new Upload({
             client,
             params: {
-                Bucket: process.env.S3_BUCKET,
+                Bucket: cfg.S3_BUCKET,
                 Key: filename,
                 Body: ndjsonStream,
                 ContentType: "application/x-ndjson",