background/ticker.go

@@ -10,6 +10,7 @@ import (
 	"git.fjla.uk/owlboard/timetable-mgr/corpus"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"git.fjla.uk/owlboard/timetable-mgr/pis"
 	"git.fjla.uk/owlboard/timetable-mgr/stations"
 	"go.uber.org/zap"
 )
@@ -42,6 +43,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
 			go cif.CheckCif(cfg)
 			go corpus.CheckCorpus(cfg)
 			go stations.Check()
+			go pis.Check()
 		}
 	}
 }

pis/check.go

@@ -61,7 +61,7 @@ func Check() {
 
 	log.Info("PIS Data being updated")
 	err = runUpdate(apiResp.TarballUrl)
-	if err == nil {
+	if err != nil {
 		log.Error("Error updating PIS Data", zap.Error(err))
 	}
 }

pis/data.go

@@ -2,19 +2,23 @@ package pis
 
 import (
 	"fmt"
+	"sort"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
 
 // Process the YAML data to a struct
-func processYaml(yamlStr string) (*PisData, error) {
-	var pis PisData
+func processYaml(yamlStr string) (*[]PisData, error) {
+	var pis []PisData
 
+	// Unmarshal the YAML data into a slice of PisData
 	err := yaml.Unmarshal([]byte(yamlStr), &pis)
 	if err != nil {
 		return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
 	}
 
+	// Perform deduplication on the entire pis slice
 	err = deduplicateCodes(&pis)
 	if err != nil {
 		return nil, err
@@ -24,6 +28,25 @@ func processYaml(yamlStr string) (*PisData, error) {
 }
 
 // Deduplicate data in place and return error if failed
-func deduplicateCodes(pis *PisData) error {
-	return fmt.Errorf("deduplication logic not present, unable to update")
+func deduplicateCodes(pis *[]PisData) error {
+	uniqueStops := make(map[string]bool)
+	var dedupedPis []PisData
+
+	for _, data := range *pis {
+		stopsKey := stopsToString(data.Stops)
+
+		// If stopsKey does not exist, add to map
+		if _, exists := uniqueStops[stopsKey]; !exists {
+			uniqueStops[stopsKey] = true
+			dedupedPis = append(dedupedPis, data)
+		}
+	}
+
+	*pis = dedupedPis
+	return nil
+}
+
+func stopsToString(stops []string) string {
+	sort.Strings(stops)
+	return strings.Join(stops, ",")
 }

pis/update.go

@@ -42,7 +42,13 @@ func runUpdate(tarballUrl string) error {
 		return err
 	}
 
-	// Replace database collection with new data
+	pisSlice, err := processYaml(pisData)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(&pisSlice)
+	// Replace db contents with new pis data
 	// Ensure indeces are present
 
 	// Cleanup files