Rename problematic files

.gitea/workflows/release.yaml

@@ -17,14 +17,9 @@ jobs:
       id: get_version
       run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
 
-    - uses: bufbuild/buf-action@v1.3
-      with:
-        setup_only: true
-        version: '1.63.0'
-    
     - uses: actions/setup-go@v5
       with:
-        go-version: '1.23'
+        go-version: '1.24'
     
     - uses: actions/setup-node@v6
       with:
@@ -32,73 +27,75 @@ jobs:
         registry-url: 'https://git.fjla.uk/api/packages/owlboard/npm'
         scope: '@owlboard'
 
-    - name: Install Go Protoc Plugin
+    - name: Install Generators
       run: |
-        go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
+        npm install -g json-schema-to-typescript typescript
+        go install github.com/atombender/go-jsonschema@latest
         echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
 
-    - name: Install TS-Proto Plugin
-      run: |
-        npm install ts-proto typescript
-        echo "$PATH:$(pwd)/ts-proto" >> $GITHUB_PATH
-
-    - name: Generate Code
-      run: buf generate
+    - run: bash scripts/build.sh
 
     - name: Build and Publish TS
       working-directory: gen/ts
       run: |
         npm init -y
         
-        # 1. Create a dynamic index.ts using Namespace exports
-        # This converts 'rail_backend/v1/pis_mapping.ts' 
-        # into 'export * as PisMapping from "./rail_backend/v1/pis_mapping";'
-        # We use awk to handle the naming conversion (snake_case to PascalCase-ish)
-        find . -name "*.ts" -not -name "index.ts" | sed 's|^\./||; s|\.ts$||' | awk -F'/' '{ 
-          name=$NF; 
-          gsub(/_/, "", name); 
-          printf "export * as %s from \"./%s\";\n", toupper(substr(name,1,1)) substr(name,2), $0 
-        }' > index.ts
+        # Build index.ts
+        echo "// Auto-generated" > index.ts
+        find . -maxdepth 1 -name "*.ts" -not -name "index.ts" | sed 's|^\./||; s|\.ts$||' | awk '{ 
+          # Use gsub to turn hyphens into underscores so we can split easily
+          clean = $0;
+          gsub(/-/, "_", clean);
           
-        # 2. Update package.json
-        jq '.name = "@owlboard/backend-data-contracts" | 
-            .version = "${{ steps.get_version.outputs.VERSION }}" |
-            .type = "module" |
-            .main = "./dist/index.js" |
-            .types = "./dist/index.d.ts" |
-            .publishConfig = { "registry": "https://git.fjla.uk/api/packages/owlboard/npm" }' \
-            package.json > temp.json && mv temp.json package.json
+          n = split(clean, parts, "_");
+          name = "";
+          for (i=1; i<=n; i++) {
+            if (length(parts[i]) > 0) {
+              name = name toupper(substr(parts[i],1,1)) substr(parts[i],2);
+            }
+          }
+          # name will now be 'DataIngressPisData' (valid TS)
+          printf "export * as %s from \"./%s.js\";\n", name, $0 
+        }' >> index.ts
 
-        # 3. Compile
-        npx tsc index.ts --declaration --module esnext --target es2022 --moduleResolution node --outDir dist/
+        VERSION="${{ steps.get_version.outputs.VERSION }}"
+        jq --arg ver "$VERSION" \
+           --arg name "@owlboard/backend-data-contracts" \
+           '.name = $name | .version = $ver | .type = "module" | .main = "./dist/index.js" | .types = "./dist/index.d.ts"' \
+           package.json > package.json.new && mv package.json.new package.json
         
+        # Compile
+        npx tsc index.ts --declaration --module nodenext --target es2022 --moduleResolution nodenext --outDir dist/ --skipLibCheck true
+        
+        # Publish
+        npm config set //git.fjla.uk/api/packages/owlboard/npm/:_authToken ${{ secrets.PACKAGE_PUSH }}
         npm publish
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.PACKAGE_PUSH }}
 
     - name: Publish Go
       run: |
-        # 1. Setup variables
-        VERSION="${{ steps.get_version.outputs.VERSION }}"
+        VERSION="v${{ steps.get_version.outputs.VERSION }}"
         MOD_NAME="git.fjla.uk/owlboard/backend-data-contracts"
-        # Create a temporary directory structure that matches Go proxy requirements
-        DEST_DIR="temp_zip/$MOD_NAME@$VERSION"
+        ZIP_ROOT="/tmp/go_upload"
+        FULL_PATH="$ZIP_ROOT/$MOD_NAME@$VERSION"
 
-        # 2. Generate go.mod and tidy
+        # 1. Prepare
         cd gen/go
-        go mod init $MOD_NAME
-        go mod tidy
         
-        # 3. Move files into the required nested structure
-        mkdir -p "../../$DEST_DIR"
-        cp -r . "../../$DEST_DIR/"
+        # 2. Initialize the module
+        go mod init "$MOD_NAME"
         
-        # 4. Zip from the temp root so the internal paths are correct
-        cd ../../temp_zip
-        zip -r ../module.zip .
+        # 3. Create the structure
+        mkdir -p "$FULL_PATH"
         
-        # 5. Upload with the explicit version
-        cd ..
-        curl -f -v --user "owlbot:${{ secrets.PACKAGE_PUSH }}" \
-          --upload-file module.zip \
+        # 4. Copy the CONTENTS of models to the root of the module
+        # This flattens the structure so the .go files are next to go.mod
+        cp -r models/* "$FULL_PATH/"
+        cp go.mod "$FULL_PATH/"
+        
+        # 5. Zip and Upload
+        cd "$ZIP_ROOT"
+        zip -r -D "$GITHUB_WORKSPACE/module.zip" .
+        
+        curl -f --user "owlbot:${{ secrets.PACKAGE_PUSH }}" \
+          --upload-file "$GITHUB_WORKSPACE/module.zip" \
           "${{ github.server_url }}/api/packages/owlboard/go/upload?version=$VERSION"

README.md

@@ -1,32 +1,18 @@
 # backend-data-contracts
 
-This repository is the single source of truth for all Protocol Buffer (Protobuf) schema definitions used across the Owlboard Rail Ingress and Processing microservices. It follows a **Contract-First** approach, where language-specific code (Go, TypeScript) is generated and distributed via private registries.
-
-## Purpose
-
-The Protobuf files defined here serve as the immutable data contract for:
-1.  **Ingress Logic:** Ensuring the Node.js and Go inregrate properly by sharing types.
-2.  **Message Payloads:** Defining messages for cross-service communication.
-3.  **Persistence:** Defining the expected structure of documents in MongoDB for the Go processing services.
+This repository is the single source of truth for all schema definitions used across the Owlboard backend communication and storage services.  Language specific types are generated here and published to the Gitea package repository linked to the repo.
 
 ## Directory Structure
 
-| Path | Description | Contents |
-| :--- | :--- | :--- |
-| `proto/` | **Source of Truth.** Contains the Protobuf schema definitions. | `rail_backend/v1/*.proto` |
-| `buf.yaml` | **Workspace Config.** Defines linting and breaking change rules (Buf v2). | Workspace settings |
-| `buf.gen.yaml` | **Generation Config.** Defines how Go and TS code is built. | Plugin & Managed Mode settings |
-| `gen/` | **Transient Output.** Generated code resides here during CI/CD. | `.pb.go`, `.js`, `.d.ts` (**Git Ignored**) |
+| Path | Description |
+| :--- | :--- |
+| `schemas` | JSON Schema files organised into clear subfolders |
+| `scripts` | Workflow Scripts |
 
 ## Code Generation and Publishing Workflow
 
 The generation and release process is automated via Gitea Actions. It is triggered whenever a new **SemVer tag** (e.g., `v1.0.0`) is pushed to this repository.
 
-1.  **Validation:** `buf lint` ensures schemas follow best practices.
-2.  **Generation:** `buf generate` creates Go and TypeScript code using local plugins.
-3.  **TS Release:** Compiles `.ts` to `.js` and publishes to the Gitea NPM Registry as `@owlboard/backend-data-contracts`.
-4.  **Go Release:** Initializes a `go.mod`, zips the artifacts, and pushes them to the Gitea Go Package Registry.
-
 ## To Consume the Contracts
 
 ### 1. Go Services
@@ -34,4 +20,32 @@ Since the package is hosted in the Gitea Package Registry, you must configure yo
 
 **Setup:**
 ```bash
-export GOPROXY=[https://git.fjla.uk/api/packages/owlboard/go,https://proxy.golang.org,direct](https://git.fjla.uk/api/packages/owlboard/go,https://proxy.golang.org,direct)
+# Set the proxy to check our Gitea instance first
+go env -w GOPROXY="https://git.fjla.uk/api/packages/OwlBoard/go,https://proxy.golang.org,direct"
+
+# Trust our internal domain and skip the public Google checksum database
+go env -w GONOSUMDB="git.fjla.uk"
+```
+
+The, you can simply run go get, as usual.
+
+#### Container Usage
+```bash
+ARG GOPROXY=https://git.fjla.uk/api/packages/OwlBoard/go,https://proxy.golang.org,direct
+ARG GONOSUMDB=git.fjla.uk
+ENV GOPROXY=$GOPROXY
+ENV GONOSUMDB=$GONOSUMDB
+```
+
+### 2. Typescript Services
+You will need to configure .npmrc in your projects root directory to point to the correct repo:
+
+```bash
+@owlboard:registry=https://git.fjla.uk/api/packages/OwlBoard/npm/
+```
+
+Then you can install as usual:
+
+```bash
+npm install @owlboard/backend-data-contracts@0.1.0
+```

buf.gen.yaml

@@ -1,16 +0,0 @@
-version: v2
-managed:
-  enabled: true
-  override:
-    - file_option: go_package_prefix
-      value: github.com/owlboard/backend-data-contracts
-plugins:
-  - local: protoc-gen-go
-    out: gen/go
-    opt: paths=source_relative
-  - local: ./node_modules/ts-proto/protoc-gen-ts_proto
-    out: gen/ts
-    opt:
-    - esModuleInterop=true
-    - outputJsonMethods=true
-    - forceLong=string

buf.yaml

@@ -1,9 +0,0 @@
-version: v2
-modules:
-  - path: protos
-lint:
-  use:
-    - DEFAULT
-breaking:
-  use:
-    - FILE

protos/rail_backend/v1/common.proto

@@ -1,10 +0,0 @@
-syntax = "proto3";
-
-package rail_backend.v1;
-
-message Metadata {
-    int64 push_to_queue_time = 1;
-    int64 data_fetch_time = 2;
-    map<string, string> tags = 3;
-}
-

protos/rail_backend/v1/pis_schema.proto

@@ -1,16 +0,0 @@
-syntax = "proto3";
-
-package rail_backend.v1;
-
-message PisReferenceList {
-    repeated PisMapping entries = 1;
-}
-
-message PisMapping {
-    string code = 1;
-    string toc = 2;
-    repeated string crsStops = 3;
-    fixed64 crsHash = 4; // XXH4 Hash for fast lookup of exact match
-    repeated string tiplocStops = 5;
-    fixed64 tiplocHash = 6;
-}

protos/rail_backend/v1/queue_message.proto

@@ -1,20 +0,0 @@
-syntax = "proto3";
-
-package rail_backend.v1;
-
-import "rail_backend/v1/common.proto";
-import "rail_backend/v1/schedule_payload.proto";
-
-message IngressMessage {
-    string correlation_id = 1;
-    Metadata tracking_data = 2;
-    oneof payload {
-        UrlReference url_ref = 5;
-        SchedulePayload schedule_payload = 6;
-    }
-}
-
-message UrlReference {
-    string kind = 1;
-    string url = 2;
-}

schemas/data-ingress/message-envelope.json

@@ -0,0 +1,32 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/message-envelope.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "MessageEnvelope",
+    "type": "object",
+    "properties": {
+        "service_name": {
+            "type": "string",
+            "description": "Name of the service submitting the update"
+        },
+        "service_id": {
+            "type": "string",
+            "description": "ID of the service sending the message - usually a hostname"
+        },
+        "data_type": {
+            "type": "string",
+            "enum": ["file"],
+            "description": "The type of data contained in this message object"
+        },
+        "sent_timestamp": {
+            "type": "integer",
+            "description": "Unix timestamp representing the time the message was sent"
+        },
+        "data_kind": {
+            "type": "string",
+            "enum": ["pis", "timetable", "knowledgebase", "darwin_schedule", "darwin_ts", "darwin_ref"],
+            "description": "The data type contained in the message.  Currently supported PIS: PIS Data, Timetable: CIF or VSTP Data, Knowledgebase: Station Data"
+        }
+    },
+    "required": ["service_name", "service_id", "data_type", "sent_timestamp"],
+    "additionalProperties": true
+}

schemas/data-ingress/mq-file-update.json

@@ -0,0 +1,47 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/mq-file-update.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "MQFileUpdate",
+    "type": "object",
+    "properties": {
+        "service_name": {
+            "type": "string",
+            "description": "Name of the service submitting the update"
+        },
+        "service_id": {
+            "type": "string",
+            "description": "ID of the service sending the message - usually a hostname"
+        },
+        "data_type": {
+            "type": "string",
+            "enum": ["file"],
+            "description": "The type of data contained in this message object"
+        },
+        "sent_timestamp": {
+            "type": "integer",
+            "description": "Unix timestamp representing the time the message was sent"
+        },
+        "data_kind": {
+            "type": "string",
+            "enum": ["pis", "timetable", "knowledgebase"],
+            "description": "The data type contained in the file.  Currently supported PIS: PIS Data, Timetable: CIF or VSTP Data, Knowledgebase: Station Data"
+        },
+        "payload": {
+            "type": "object",
+            "properties": {
+                "version": {
+                    "type": "string",
+                    "description": "The version string from the package source"
+                },
+                "filepath": {
+                    "type": "string",
+                    "description": "The full path to the file, including protocol (eg. s3://) where appropriate"
+                }
+            },
+            "required": ["version", "filepath"],
+            "additionalProperties": false
+        }
+    },
+    "required": ["service_name", "service_id", "data_type", "sent_timestamp", "data_kind", "payload"],
+    "additionalProperties": false
+}

schemas/data-ingress/pis-data.json

@@ -0,0 +1,54 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/pis-data.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "PisObjects",
+    "type": "object",
+    "properties": {
+        "code": {
+            "type": "string",
+            "description": "PIS Code - Code that is entered in to the PIS system"
+        },
+        "toc": {
+            "type": "string",
+            "minLength": 2,
+            "maxLength": 2,
+            "pattern": "^[a-zA-Z]+$",
+            "description": "Two letter TOC Code"
+        },
+        "crsStops": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "minLength": 3,
+                "maxLength": 3,
+                "pattern": "^[a-zA-Z]+$"
+            },
+            "description": "List of 3ALPHA/CRS Codes"
+        },
+        "crsHash": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 64,
+            "pattern": "^[0-9]+$",
+            "description": "Stringified 64-bit hash"
+        },
+        "tiplocStops": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "minLength": 4,
+                "maxLength": 7,
+                "pattern": "^[a-zA-Z0-9]+$"
+            },
+            "description": "List of TIPLOC Codes"
+        },
+        "tiplocHash": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 64,
+            "pattern": "^[0-9]+$"
+        }
+    },
+    "required": ["code", "toc", "crsStops", "crsHash", "tiplocStops", "tiplocHash"],
+    "additionalProperties": false
+}

schemas/data-ingress/pushport/schedule.json

@@ -0,0 +1,39 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/pushport-schedule.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "PushPortSchedule",
+    "type": "object",
+    "required": ["rid", "uid", "trainId", "ssd", "toc", "locations"],
+    "properties": {
+        "rid": {"type": "string", "description": "Unique service ID"},
+        "uid": {"type": "string", "description": "Unique schedule ID - maps to NR CIF"},
+        "trainId": {"type": "string", "description": "Train Headcode"},
+        "ssd": {"type": "string", "format": "date", "description": "Schedule Start Date (When the service commences)"},
+        "toc": {"type": "string", "minLength": 2, "maxLength": 2, "description": "TOC Code"},
+        "isPassenger": {"type": "boolean", "description": "If service is for passengers", "default": true},
+        "isCharter": {"type": "boolean", "description": "If train is charter service", "default": false},
+        "status": {"type": "string", "enum": ["train", "bus", "ship"], "default": "train"},
+        "isDelayedDelivery": {"type": "boolean", "description": "If message is sent as part of Darwin service recovery"},
+        "cancelReason": {"type": ["string", "null"], "description": "Reason code for cancellation"},
+        "isDeleted": {"type": "boolean", "description": "Whether schedule should be deleted", "default": false},
+        "locations": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": ["tpl", "seq"],
+                "properties": {
+                    "tpl": {"type": "string", "description": "TIPLOC of location"},
+                    "seq": {"type": "integer", "description": "The order of the stop for sorting and handling repeated TIPLOCs"},
+                    "wta": { "type": ["string", "null"] },
+                    "wtd": { "type": ["string", "null"] },
+                      "wtp": { "type": ["string", "null"] },
+                      "pta": { "type": ["string", "null"] },
+                      "ptd": { "type": ["string", "null"] },
+                      "plat": { "type": ["string", "null"] },
+                      "isCancelled": { "type": "boolean", "default": false },
+                      "activities": { "type": "string" }
+                }
+            }
+        }
+    }
+}

schemas/data-ingress/pushport/ts.json.txt

@@ -0,0 +1,39 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/pushport-ts.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "PushPortTS",
+    "type": "object",
+    "properties": {
+        "rid": {
+            "type": "string",
+            "description": "Unique ID for the train service (eg. 2026022217610103)",
+            "pattern": "^[0-9]{15}$"
+        },
+        "uid": {
+            "type": "string",
+            "description": "Schedule ID - The permanent ID for the schedule in the static timetable"
+        },
+        "ssd": {
+            "type": "string",
+            "format": "date",
+            "description": "Scheduled Start Data, the date the train service started its journey"
+        },
+        "isDelayed": {
+            "type": "boolean",
+            "description": "Indicated if the message is delayed due to PushPort service disruption"
+        },
+        "lateReason": {
+            "type": "integer",
+            "description": "Reason the train is late (Maps to textual reason)"
+        },
+        "cancelReason": {
+            "type": "integer",
+            "description": "Reason the train is cancelled (Maps to textual reason)"
+        },
+        "locations": {
+            "type": "array",
+            "description": "A list of one or more locations where the train has recorded a movement or forecast",
+            "items": { "$ref": "https://schema.owlboard.info/data-ingress/pushport-ts-location.schema.json" }
+        }
+    }
+}

schemas/data-ingress/pushport/ts_location.json.txt

@@ -0,0 +1,51 @@
+{
+    "$id": "https://schema.owlboard.info/data-ingress/pushport-ts-location.schema.json",
+    "$schema": "https://json-schema.org/draft-07/schema#",
+    "title": "PushPortTSLocation",
+    "type": "object",
+    "required": ["tpl"],
+    "definitions": {
+        "TSTime": {
+            "type": "object",
+            "properties": {
+                "at": {"type": "string", "description": "Actual time"},
+                "et": {"type": "string", "description": "Estimated time"},
+                "wet": {"type": "string", "description": "Working estimated time"},
+                "src": {"type": "string", "description": "Data source (TD/Darwin/etc)"},
+                "atClass": {"type": "string", "description": "Time source Automatic/Manual"},
+                "delayed": {"type": "boolean", "description": "is train delayed with no estimated time"},
+                "srcInst": {"type": "string", "description": "Source instance (what caused the update)"},
+                "atRemoved": {"type": "boolean", "description": "Whether a previous 'at' was reported in error and should be removed"}
+            }
+        }
+    },
+    "properties": {
+        "tpl": {"type": "string", "description": "TIPLOC for the location"},
+        "wta": {"type": "string", "description": "Scheduled working time arrival"},
+        "wtd": {"type": "string", "description": "Scheduled working time departure"},
+        "wtp": {"type": "string", "description": "Scheduled working time pass"},
+        "pta": {"type": "string", "description": "Scheduled public arrival time"},
+        "ptd": {"type": "string", "description": "Scheduled public departure time"},
+        "arr": {"$ref": "#/definitions/TSTime"},
+        "dep": {"$ref": "#/definitions/TSTime"},
+        "pass": {"$ref": "#/definitions/TSTime"},
+        "plat": {
+            "type": "object",
+            "description": "Platform data",
+            "properties": {
+                "value": {"type": "string", "description": "Platform number/letter"},
+                "platsrc": {"type": "string", "enum": ["A", "M", "P"], "description": "Source of data Automatic/Manual/Planned"},
+                "platsup": {"type": "boolean", "description": "Is platform suppressed from public display"},
+                "cisPlatsup": {"type": "boolean", "description": "Whether CIS or Darwin Workstation has supressed the platform"},
+                "conf": {"type": "boolean", "description": "Whether platform has been confirmed by the local CIS"}
+            }
+        },
+        "suppr": {"type": "boolean", "description": "Whether location is supressed"},
+        "length": {"type": "integer", "description": "Train length (no. of coaches)"},
+        "detachFront": {"type": "boolean","description": "Indicated if coaches will detatch from front"},
+        "detachRead": {"type": "boolean", "description": "Indicates if coaches will detatch from rear"},
+        "attachFront": {"type": "boolean", "description": "Indicates if coaches will attach to front"},
+        "attachRear": {"type": "boolean", "description": "Indicates if coaches will be attached to rear"},
+        "adhoc": {"type": "boolean", "description": "Whether this is an ad-hoc stop"}
+    }
+}

scripts/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# Create clean output directories
+rm -rf gen && mkdir -p gen/ts gen/go/models
+
+# Find all .json files
+FILES=$(find schemas -name "*.json")
+
+# Initialize the TypeScript Barrel File
+echo "// Auto-generated barrel file" > gen/ts/index.ts
+
+for file in $FILES; do
+    # Get a clean name (e.g., data-ingress_pis-mapping)
+    clean_name=$(echo "${file#schemas/}" | sed 's/\//_/g' | sed 's/\.json//g')
+
+    # OGenerate TS
+    npx --yes json-schema-to-typescript "$file" > "gen/ts/${clean_name}.ts"
+    
+    # Generate Go
+    go-jsonschema -p contracts "$file" > "gen/go/models/${clean_name}.go"
+done
+
+echo "✅ Generated single TS package in gen/ts"
+echo "✅ Generated single Go package in gen/go/models"