diff --git a/background/ticker.go b/background/ticker.go index e47bf74..9153757 100644 --- a/background/ticker.go +++ b/background/ticker.go @@ -10,6 +10,7 @@ import ( "git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" + "git.fjla.uk/owlboard/timetable-mgr/pis" "git.fjla.uk/owlboard/timetable-mgr/stations" "go.uber.org/zap" ) @@ -42,6 +43,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) { go cif.CheckCif(cfg) go corpus.CheckCorpus(cfg) go stations.Check() + go pis.Check() } } } diff --git a/pis/check.go b/pis/check.go index 2aff9b1..034742b 100644 --- a/pis/check.go +++ b/pis/check.go @@ -61,7 +61,7 @@ func Check() { log.Info("PIS Data being updated") err = runUpdate(apiResp.TarballUrl) - if err == nil { + if err != nil { log.Error("Error updating PIS Data", zap.Error(err)) } } diff --git a/pis/data.go b/pis/data.go index 483a843..b75e54a 100644 --- a/pis/data.go +++ b/pis/data.go @@ -2,19 +2,23 @@ package pis import ( "fmt" + "sort" + "strings" "gopkg.in/yaml.v3" ) // Process the YAML data to a struct -func processYaml(yamlStr string) (*PisData, error) { - var pis PisData +func processYaml(yamlStr string) (*[]PisData, error) { + var pis []PisData + // Unmarshal the YAML data into a slice of PisData err := yaml.Unmarshal([]byte(yamlStr), &pis) if err != nil { return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) } + // Perform deduplication on the entire pis slice err = deduplicateCodes(&pis) if err != nil { return nil, err @@ -24,6 +28,25 @@ func processYaml(yamlStr string) (*PisData, error) { } // Deduplicate data in place and return error if failed -func deduplicateCodes(pis *PisData) error { - return fmt.Errorf("deduplication logic not present, unable to update") +func deduplicateCodes(pis *[]PisData) error { + uniqueStops := make(map[string]bool) + var dedupedPis []PisData + + for _, data := range *pis { + stopsKey := stopsToString(data.Stops) + + // If stopsKey does not exist, add to map + if _, exists := uniqueStops[stopsKey]; !exists { + uniqueStops[stopsKey] = true + dedupedPis = append(dedupedPis, data) + } + } + + *pis = dedupedPis + return nil +} + +func stopsToString(stops []string) string { + sort.Strings(stops) + return strings.Join(stops, ",") } diff --git a/pis/update.go b/pis/update.go index 1ec6a32..395988e 100644 --- a/pis/update.go +++ b/pis/update.go @@ -42,7 +42,13 @@ func runUpdate(tarballUrl string) error { return err } - // Replace database collection with new data + pisSlice, err := processYaml(pisData) + if err != nil { + return err + } + + fmt.Println(&pisSlice) + // Replace db contents with new pis data // Ensure indeces are present // Cleanup files