Implement PIS task
This commit is contained in:
parent
8f0e9759c4
commit
0881be7a79
@ -10,6 +10,7 @@ import (
|
|||||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/pis"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
@ -42,6 +43,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
|||||||
go cif.CheckCif(cfg)
|
go cif.CheckCif(cfg)
|
||||||
go corpus.CheckCorpus(cfg)
|
go corpus.CheckCorpus(cfg)
|
||||||
go stations.Check()
|
go stations.Check()
|
||||||
|
go pis.Check()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -61,7 +61,7 @@ func Check() {
|
|||||||
|
|
||||||
log.Info("PIS Data being updated")
|
log.Info("PIS Data being updated")
|
||||||
err = runUpdate(apiResp.TarballUrl)
|
err = runUpdate(apiResp.TarballUrl)
|
||||||
if err == nil {
|
if err != nil {
|
||||||
log.Error("Error updating PIS Data", zap.Error(err))
|
log.Error("Error updating PIS Data", zap.Error(err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
31
pis/data.go
31
pis/data.go
@ -2,19 +2,23 @@ package pis
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Process the YAML data to a struct
|
// Process the YAML data to a struct
|
||||||
func processYaml(yamlStr string) (*PisData, error) {
|
func processYaml(yamlStr string) (*[]PisData, error) {
|
||||||
var pis PisData
|
var pis []PisData
|
||||||
|
|
||||||
|
// Unmarshal the YAML data into a slice of PisData
|
||||||
err := yaml.Unmarshal([]byte(yamlStr), &pis)
|
err := yaml.Unmarshal([]byte(yamlStr), &pis)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
|
return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Perform deduplication on the entire pis slice
|
||||||
err = deduplicateCodes(&pis)
|
err = deduplicateCodes(&pis)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -24,6 +28,25 @@ func processYaml(yamlStr string) (*PisData, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Deduplicate data in place and return error if failed
|
// Deduplicate data in place and return error if failed
|
||||||
func deduplicateCodes(pis *PisData) error {
|
func deduplicateCodes(pis *[]PisData) error {
|
||||||
return fmt.Errorf("deduplication logic not present, unable to update")
|
uniqueStops := make(map[string]bool)
|
||||||
|
var dedupedPis []PisData
|
||||||
|
|
||||||
|
for _, data := range *pis {
|
||||||
|
stopsKey := stopsToString(data.Stops)
|
||||||
|
|
||||||
|
// If stopsKey does not exist, add to map
|
||||||
|
if _, exists := uniqueStops[stopsKey]; !exists {
|
||||||
|
uniqueStops[stopsKey] = true
|
||||||
|
dedupedPis = append(dedupedPis, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
*pis = dedupedPis
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopsToString(stops []string) string {
|
||||||
|
sort.Strings(stops)
|
||||||
|
return strings.Join(stops, ",")
|
||||||
}
|
}
|
||||||
|
@ -42,7 +42,13 @@ func runUpdate(tarballUrl string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Replace database collection with new data
|
pisSlice, err := processYaml(pisData)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(&pisSlice)
|
||||||
|
// Replace db contents with new pis data
|
||||||
// Ensure indeces are present
|
// Ensure indeces are present
|
||||||
|
|
||||||
// Cleanup files
|
// Cleanup files
|
||||||
|
Loading…
Reference in New Issue
Block a user