diff --git a/dbAccess/contants.go b/dbAccess/contants.go index d0bd67d..4e02211 100644 --- a/dbAccess/contants.go +++ b/dbAccess/contants.go @@ -5,3 +5,4 @@ const CorpusCollection string = "corpus" const StationsCollection string = "stations" const MetaCollection string = "meta" const TimetableCollection string = "timetable" +const PisCollection string = "pis" diff --git a/dbAccess/pis.go b/dbAccess/pis.go index 0d024fb..bb81a52 100644 --- a/dbAccess/pis.go +++ b/dbAccess/pis.go @@ -3,10 +3,14 @@ package dbAccess import ( "context" "errors" + "time" "git.fjla.uk/owlboard/go-types/pkg/database" + "git.fjla.uk/owlboard/timetable-mgr/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" ) func GetPisMetadata() (*database.PisMetadata, error) { @@ -24,3 +28,61 @@ func GetPisMetadata() (*database.PisMetadata, error) { return &result, nil } + +func PutPisMetadata(version string) error { + coll := MongoClient.Database(DatabaseName).Collection(MetaCollection) + options := options.Update().SetUpsert(true) + filter := bson.M{"type": "PisMetadata"} + update := bson.M{ + "$set": bson.M{ + "type": "PisMetadata", + "lastUpdate": time.Now().Format(time.RFC3339), + "lastVersion": version, + }, + } + + _, err := coll.UpdateOne(context.Background(), filter, update, options) + + if err != nil { + return err + } + + log.Info("New Stations Metadata written", zap.String("version", version)) + return nil +} + +// Puts complete PIS dataset to database +func PutPisData(pis *[]database.PIS) (int64, error) { + coll := MongoClient.Database(DatabaseName).Collection(PisCollection) + + var docs []interface{} + for _, entry := range *pis { + docs = append(docs, entry) + } + + res, err := coll.InsertMany(context.TODO(), docs) + if err != nil { + return 0, err + } + + return int64(len(res.InsertedIDs)), nil +} + +func CreatePisIndeces() error { + coll := MongoClient.Database(DatabaseName).Collection(PisCollection) + + crsIndex := mongo.IndexModel{ + Keys: bson.D{{"stops", 1}}, + } + + tiplocIndex := mongo.IndexModel{ + Keys: bson.D{{"tiplocs", 1}}, + } + + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{crsIndex, tiplocIndex}) + if err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 6831ad3..849102d 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,23 @@ go 1.22 toolchain go1.22.4 require ( - git.fjla.uk/owlboard/go-types v1.1.10 - github.com/go-stomp/stomp/v3 v3.1.0 - go.mongodb.org/mongo-driver v1.16.0 + git.fjla.uk/owlboard/go-types v1.1.11 + github.com/go-stomp/stomp/v3 v3.1.2 + go.mongodb.org/mongo-driver v1.17.1 go.uber.org/zap v1.27.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/text v0.16.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 50f26f9..d56c01d 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,23 @@ -git.fjla.uk/owlboard/go-types v1.1.9 h1:iZubatvgZZBNckVrAjkEyRU7lnx71dG73mA7bb2ePo8= -git.fjla.uk/owlboard/go-types v1.1.9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= git.fjla.uk/owlboard/go-types v1.1.10 h1:r7XMJ6TzaNomv0HmUrNkPd8ce09c4pYd6D9Bh7M0xwY= git.fjla.uk/owlboard/go-types v1.1.10/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= +git.fjla.uk/owlboard/go-types v1.1.11 h1:EKIPcSHymmiTBTj5/NmSoe7ycv9YnUK2hjriRmWJl7Y= +git.fjla.uk/owlboard/go-types v1.1.11/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-stomp/stomp/v3 v3.1.0 h1:JnvRJuua/fX2Lq5Ie5DXzrOL18dnzIUenCZXM6rr8/0= github.com/go-stomp/stomp/v3 v3.1.0/go.mod h1:ztzZej6T2W4Y6FlD+Tb5n7HQP3/O5UNQiuC169pIp10= +github.com/go-stomp/stomp/v3 v3.1.2 h1:kmrNek021BsFUO8rxDhbkOYslRomKO/JIrUCIqyL0r8= +github.com/go-stomp/stomp/v3 v3.1.2/go.mod h1:ztzZej6T2W4Y6FlD+Tb5n7HQP3/O5UNQiuC169pIp10= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -34,16 +36,16 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 h1:tBiBTKHnIjovYoLX/TPkcf+OjqqKGQrPtGT3Foz+Pgo= github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76/go.mod h1:SQliXeA7Dhkt//vS29v3zpbEwoa+zb2Cn5xj5uO4K5U= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -52,12 +54,11 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -70,6 +71,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -84,17 +87,16 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/helpers/config.go b/helpers/config.go index 270485f..b02ce71 100644 --- a/helpers/config.go +++ b/helpers/config.go @@ -5,7 +5,7 @@ import ( ) // Version Constants -const versionNum string = "2024.10.0" +const versionNum string = "2024.10.1" const versionSuffix string = "" const Version string = versionNum + versionSuffix diff --git a/pis/data.go b/pis/data.go index b75e54a..2e73044 100644 --- a/pis/data.go +++ b/pis/data.go @@ -5,11 +5,15 @@ import ( "sort" "strings" + "git.fjla.uk/owlboard/go-types/pkg/database" + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.uber.org/zap" "gopkg.in/yaml.v3" ) // Process the YAML data to a struct -func processYaml(yamlStr string) (*[]PisData, error) { +func processYaml(yamlStr string) (*[]database.PIS, error) { var pis []PisData // Unmarshal the YAML data into a slice of PisData @@ -24,7 +28,9 @@ func processYaml(yamlStr string) (*[]PisData, error) { return nil, err } - return &pis, nil + documents, err := convertPisForDatabase(&pis) + + return documents, nil } // Deduplicate data in place and return error if failed @@ -46,7 +52,42 @@ func deduplicateCodes(pis *[]PisData) error { return nil } +// Join slice elements to a string for comparison func stopsToString(stops []string) string { sort.Strings(stops) return strings.Join(stops, ",") } + +// Convert PisData to database.PIS +func convertPisForDatabase(in *[]PisData) (*[]database.PIS, error) { + var out []database.PIS + + for _, code := range *in { + var document database.PIS + document.Code = code.Code + document.Operator = code.Operator + document.Stops = code.Stops + document.Tiplocs = GetTiplocsFromCrs(code.Stops) + + out = append(out, document) + } + + return &out, nil +} + +// Return a list of TIPLOCS from a list of CRS using Database Lookups +func GetTiplocsFromCrs(crsList []string) (TiplocList []string) { + for _, crs := range crsList { + alpha := strings.ToUpper(crs) + tiploc, err := dbAccess.GetTiplocFromCrs(alpha) + + if err != nil { + log.Warn("Unable to find matching TIPLOC for 3ALPHA", zap.String("3ALPHA", alpha), zap.Error(err)) + return + } + + TiplocList = append(TiplocList, strings.ToUpper(tiploc)) + } + + return +} diff --git a/pis/update.go b/pis/update.go index 395988e..d782315 100644 --- a/pis/update.go +++ b/pis/update.go @@ -9,6 +9,10 @@ import ( "os" "path/filepath" "strings" + + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.uber.org/zap" ) const ( @@ -26,13 +30,11 @@ func runUpdate(tarballUrl string) error { // Extract to disk file, err := os.Open(destPath) if err != nil { - fmt.Printf("Error opening file: %v\n", err) return err } defer file.Close() if err := extractFiles(file, extractPath); err != nil { - fmt.Printf("Error extracting file: %v\n", err) return err } @@ -48,8 +50,23 @@ func runUpdate(tarballUrl string) error { } fmt.Println(&pisSlice) - // Replace db contents with new pis data - // Ensure indeces are present + + err = dbAccess.DropCollection(dbAccess.PisCollection) + if err != nil { + return err + } + + count, err := dbAccess.PutPisData(pisSlice) + if err != nil { + return err + } + + log.Info("Insterted new PIS Data", zap.Int64("PIS Codes", count)) + + err = dbAccess.CreatePisIndeces() + if err != nil { + log.Error("Failed to create PIS Indeces, poor performance expected", zap.Error(err)) + } // Cleanup files cleanupFiles(destPath, extractPath)