From 9ebf62e2c621d4ea21598af710dc681a28939e13 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Tue, 16 Dec 2025 21:42:16 +0000 Subject: [PATCH] Add automation --- .gitea/workflows/release.yaml | 60 ++++++++ README.md | 34 +++++ buf.gen.yaml | 11 ++ go.mod | 3 + protos/rail/v1/common.proto | 11 ++ protos/rail/v1/pis_schema.proto | 15 ++ protos/rail/v1/queue_message.proto | 22 +++ protos/rail/v1/schedule_payload.proto | 199 ++++++++++++++++++++++++++ 8 files changed, 355 insertions(+) create mode 100644 .gitea/workflows/release.yaml create mode 100644 README.md create mode 100644 buf.gen.yaml create mode 100644 go.mod create mode 100644 protos/rail/v1/common.proto create mode 100644 protos/rail/v1/pis_schema.proto create mode 100644 protos/rail/v1/queue_message.proto create mode 100644 protos/rail/v1/schedule_payload.proto diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml new file mode 100644 index 0000000..80cdfb9 --- /dev/null +++ b/.gitea/workflows/release.yaml @@ -0,0 +1,60 @@ +name: Generate and Release Protos +on: + push: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Get Version + id: get_version + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT + + - uses: bufbuild/buf-setup-action@v1 + + - name: Generate Code + run: buf generate + + - uses: actions/setup-node@v6 + with: + node-version: '18.18.x' + registry-url: 'https://git.fjla.uk/api/packages/owlboard/npm' + scope: '@owlboard' + + - name: Publish TS + working-directory: gen/ts + run: | + npm init -y + jq '.name = "@owlboard/backend-data-contracts" | + .version = "${{ steps.get_version.outputs.VERSION }}" | + .description = "Generated Protobuf types for data ingress services" | + .repository = { + "type": "git", + "url": "git+https://${{ github.server_url }}/${{ github.repository }}.git" + } | + .bugs = { + "url": "https://${{ github.server_url }}/${{ github.repository }}/issues" + } | + .homepage = "https://${{ github.server_url }}/${{ github.repository }}#readme"' \ + package.json > temp.json && mv temp.json package.json + npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.PACKAGE_PUSH }} + + - name: Commit Generated Go + run: | + git config user.name "owlbot" + git config user.email "owlbot@owlboard.info" + git add gen/go/*.go + git commit -m "OwlBot: Generated go types for ${{ steps.get_version.outputs.VERSION }}" + git diff-index --quiet HEAD || git commit -m "OwlBot: Generated go types for ${{ steps.get_version.outputs.VERSION }}" + git push origin HEAD:refs/tags/${{ github.ref_name }} --force + env: + GITHUB_TOKEN: ${{ secrets.REPO_PUSH }} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..db4bc06 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# data-contracts + +This repository is the single source of truth for all Protocol Buffer (Protobuf) schema definitions used across the Rail Ingress and Processing microservices. + +## Purpose + +The Protobuf files defined here serve as the immutable data contract for: +1. **Message Queue Payloads:** Defining messages pushed to the Artemis queue (Go Process Service consumption). +2. **Database Schemas:** Defining the expected structure of documents in MongoDB (Go Process and TypeScript API service consumption). +3. **Cross-Service Communication:** Ensuring type-safe data exchange between all polyglot services (Go, TypeScript). + +## Directory Structure and Artifacts + +| Path | Description | Contents | +| :--- | :--- | :--- | +| `protos/rail/v1/` | **Source Code.** Contains all source `.proto` files. **Only these files reside on the `main` branch.** | `.proto` files | +| `ts/` | **Generated TypeScript/JavaScript code.** Used as the root for the NPM package publish. **Not committed to Git.** | `package.json`, generated `.js`, `.d.ts` | +| `go.mod` | Defines the Go Module path: `git.fjla.uk/owlboard/backend-data-contracts`. | Go Module definition | + +## Code Generation and Publishing Workflow (Gitea Action) + +The generation process is automated via a Gitea Action (`.gitea/workflows/generate_contracts.yml`), which runs when a change is pushed to a source `.proto` file. + +The action performs the following steps: +1. Generates all Go and TypeScript/JavaScript artifacts. +2. **Go Artifacts:** Commits the generated `*.pb.go` files to a **new Git tag** (e.g., `v1.0.1`), ensuring the `main` branch remains clean. +3. **TypeScript Artifacts:** Packages the generated files and publishes the corresponding version (e.g., `1.0.1`) to the internal NPM registry. + +### To Consume the Contract: + +| Language | Artifact | Consumption Method | +| :--- | :--- | :--- | +| **Go** | Source Code (`*.pb.go`) | **Requires a Git Tag.** Update your service's `go.mod` file to reference the desired tag: `git.fjla.uk/owlboard/backend-data-contracts v1.0.1`. | +| **TypeScript** | NPM Package | Update the version in your `package.json` file and install: `"@owlboard/contracts": "1.0.1"`. | diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000..a54b88c --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,11 @@ +version: v1 +plugins: + - plugin: go + out: gen/go + opt: paths=source_relative + - plugin: ts-proto + out: gen/ts + opt: + - esModuleInterop=true + - outputJsonMethods=true + - forceLong=string \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f49f1f5 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.fjla.uk/owlboard/backend-data-contracts + +go 1.24.10 diff --git a/protos/rail/v1/common.proto b/protos/rail/v1/common.proto new file mode 100644 index 0000000..3d2365a --- /dev/null +++ b/protos/rail/v1/common.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package rail.v1; +option go_package = "git.fjla.uk/owlboard/generated/go/rail/v1"; + +message Metadata { + int64 push_to_queue_time = 1; + int64 data_fetch_time = 2; + map tags = 3; +} + diff --git a/protos/rail/v1/pis_schema.proto b/protos/rail/v1/pis_schema.proto new file mode 100644 index 0000000..9a92e47 --- /dev/null +++ b/protos/rail/v1/pis_schema.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package rail.v1; +option go_package = "git.fjla.uk/owlboard/backend-data-contracts"; + +message PisReferenceList { + repeated PisMapping entries = 1; +} + +message PisMapping { + string code = 1; + string operator = 2; + repeated string stops = 3; + fixed64 stops_xxh4 = 4; // XXH4 Hash for fast lookup of exact match +} \ No newline at end of file diff --git a/protos/rail/v1/queue_message.proto b/protos/rail/v1/queue_message.proto new file mode 100644 index 0000000..3b1ba89 --- /dev/null +++ b/protos/rail/v1/queue_message.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package rail.v1; +option go_package = "git.fjla.uk/owlboard/generated/go/rail/v1"; + + +import "rail/v1/common.proto"; +import "rail/v1/schedule_payload.proto"; + +message IngressMessage { + string correlation_id = 1; + Metadata tracking_data = 2; + oneof payload { + UrlReference url_ref = 5; + SchedulePayload schedule_payload = 6; + } +} + +message UrlReference { + string kind = 1; + string url = 2; +} \ No newline at end of file diff --git a/protos/rail/v1/schedule_payload.proto b/protos/rail/v1/schedule_payload.proto new file mode 100644 index 0000000..ca51229 --- /dev/null +++ b/protos/rail/v1/schedule_payload.proto @@ -0,0 +1,199 @@ +syntax = "proto3"; + +package rail.v1; +option go_package = "git.fjla.uk/owlboard/generated/go/rail/v1"; + +enum SchedulePayloadType { + VSTP_MESSAGE_TYPE_UNSPECIFIED = 0; + Create = 1; + Delete = 2; +} + +message RunsOnDays { + bool sunday = 1; + bool monday = 2; + bool tuesday = 3; + bool wednesday = 4; + bool thursday = 5; + bool friday = 6; + bool saturday = 7; + bool bank_holidays = 8; +} + +enum TrainStatus { + TRAIN_STATUS_UNSPECIFIED = 0; + PERMANENT_BUS = 1; + PERMANENT_FREIGHT = 2; + PERMANENT_PASSENGER_OR_PARCELS = 3; + PERMANENT_SHIP = 4; + PERMANENT_TRIP = 5; + STP_PASSENGER_ORPARCELS = 6; + STP_FREIGHT = 7; + STP_TRIP = 8; + STP_SHIP = 9; + STP_BUS = 10; +} + +enum TrainCategory { + TRAIN_CATEGORY_UNSPECIFIED = 0; + METRO_SERVICE = 1; + UNADVERTISED_PASSENGER_TRAIN = 2; + ORDINARY_PASSENGER_TRAIN = 3; + STAFF_TRAIN = 4; + MIXED_TRAIN = 5; + CHANNEL_TUNNEL_TRAIN = 6; + EUROPEAN_SLEEPER_TRAIN = 7; + INTERNATIONAL_PASSENGER_TRAIN = 8; + MOTORAIL_SERVICE = 9; + UNADVERTISED_EXPRESS_TRAIN = 10; + EXPRESS_PASSENGER_TRAIN = 11; + SLEEPER_SERVICE = 12; + REPLACEMENT_BUS_SERVICE = 13; + BUS_SERVICE = 14; + SHIP = 15; + EMPTY_COACHING_STOCK = 16; + EMPTY_METRO_SERVICE = 17; + METRO_STAFF_SERVICE = 18; + POSTAL_TRAIN = 19; + POST_OFFICE_PARCELS_TRAIN = 20; + EMPTY_NON_PASSENGER_STOCK = 21; + DEPARTMENTAL_TRAIN = 22; + CIVIL_ENGINEER_TRAIN = 23; + MECHANICAL_AND_ELECTRICAL_ENGINEER_TRAIN = 24; + STORES_TRAIN = 25; + TEST_TRAIN = 26; + SIGNAL_AND_TELECOMMUNICATIONS_ENGINEER_TRAIN = 27; + LOCOMOTIVE_AND_BRAKE_VAN = 28; + LIGHT_LOCOMOTIVE = 29; + AUTOMOTIVE_COMPONENTS_TRAIN = 30; + AUTOMOTIVE_VEHICLE_TRAIN = 31; + EDIBLE_PRODUCTS_TRAIN = 32; + INDUSTRIAL_MINERALS_TRAIN = 33; + CHEMICAL_TRAIN = 34; + BUILDING_MATERIALS_TRAIN = 35; + GENERAL_MERCHANDISE_TRAIN = 36; + EUROPEAN_RAILFREIGHT = 37; + FREIGHTLINER_CONTRACTS = 38; + FREIGHTLINER_OTHER = 39; + COAL_TRAIN = 40; + COAL_POWER_STATION_TRAIN = 41; + COAL_OR_NUCLEAR_TRAIN = 42; + METALS_TRAIN = 43; + AGGREGATES_TRAIN = 44; + DOMESTIC_AND_INDUSTRIAL_WASTE_TRAIN = 45; + TRAINLOAD_BUILDING_MATERIALS_TRAIN = 46; + PETROLEUM_PRODUCTS_TRAIN = 47; + MIXED_FREIGHT_CHANNEL_TUNNEL_TRAIN = 48; + INTERMODAL_CHANNEL_TUNNEL_TRAIN = 49; + AUTOMOTIVE_FREIGHT_CHANNEL_TUNNEL_TRAIN = 50; + CONTRACT_FREIGHT_CHANNEL_TUNNEL_TRAIN = 51; + HAULMARK_FREIGHT_CHANNEL_TUNNEL_TRAIN = 52; + JOINT_VENTURE_CHANNEL_TUNNEL_FREIGHT_TRAIN = 53; +} + +enum PowerType { + POWER_TYPE_UNSPECIFIED = 0; + DIESEL = 1; + DIESEL_MULTIPLE_UNIT = 2; + DISEL_MECHANICAL_MULTIPLE_UNIT = 3; + ELECTRIC = 4; + ELECTRO_DIESEL = 5; + EMU_PLUS_LOCOMOTIVE = 6; + ELECTRIC_MULTIPLE_UNIT = 7; + HIGH_SPEED_TRAIN = 8; +} + +message OperatingCharacteristics { + bool vacuum_braked = 1; // B + bool timed_at_100mph = 2; // C + bool driver_only_operated = 3; // D + bool conveys_mk4_coaches = 4; // E + bool guard_required = 5; // G + bool timed_at_110mph = 6; // M + bool push_pull_train = 7; // P + bool runs_as_required = 8; // Q + bool air_conditioned_with_pa = 9;//R + bool steam_heated = 10; + bool runs_to_terminals_as_required = 11; // Y + bool may_convey_sb1c_gauge = 12; // Z +} + +enum SleepingAccommodation { + NO_SLEEPING_ACCOMMODATION = 0; + FIRST_AND_STANDARD = 1; + FIRST_ONLY = 2; + STANDARD_ONLY = 3; +} + +message Reservations { + bool reservations_available = 1; + bool reservations_compulsory = 2; + bool bike_reservations_essential = 3; + bool reservations_from_any_station = 4; +} + +message Catering { + bool buffet_service = 1; + bool first_class_restaurant = 2; + bool hot_food_available = 3; + bool first_class_meal_included = 4; + bool restaurant_available = 5; + bool trolley_service = 6; +} + +enum STPIndicator { + STP_INDICATOR_NOT_SPECIFIED = 0; + C = 1; + N = 2; + O = 3; + P = 4; +} + +enum RecordIdentity { + RECORD_IDENTITY_NOT_SPECIFIED = 0; + ORIGIN = 1; + INTERMEDIATE = 2; + TERMINATING = 3; +} + +message ScheduleLocation { + RecordIdentity record_identity = 1; + string tiploc = 2; + int32 tiploc_instance = 3; + int32 arrival_offset_seconds = 4; // Seconds after midnight + int32 departure_offset_seconds = 5; + int32 pass_offset_seconds = 6; + int32 public_arrival_offset_seconds = 7; + int32 public_departure_offset_seconds = 8; + string platform = 9; + string line = 10; + string path = 11; + int32 engineering_allowance_seconds = 12; + int32 performance_allowance_seconds = 13; +} + +message SchedulePayload { + string train_uid = 1; + SchedulePayloadType transaction_type = 2; + int32 schedule_start_date = 3; // Literal int of date: eg. 20251216 + int32 schedule_end_date = 4; + RunsOnDays runs_on_day_of_week = 5; + TrainStatus train_status = 6; + TrainCategory train_category = 7; + string headcode = 8; // signalling_id + string nrs_code = 9; // CIF_headcode + string train_service_code = 10; + string portion_id = 11; // CIF_business_sector + PowerType power_type = 12; + string timing_load = 13; // Pretty String in Ingress Service + OperatingCharacteristics operating_characteristics = 14; + bool first_class_available = 15; + SleepingAccommodation sleeping_accommodation = 16; + Reservations reservations = 17; + Catering catering_code = 18; + bool is_eurostar = 19; + STPIndicator stp_indicator = 20; + string uic_code = 21; + string toc_code = 22; // Maybe ENUM but subject to frequent-ish change + repeated ScheduleLocation schedule_location = 23; +} \ No newline at end of file