Compare commits
3 Commits
6ec8cd8895
...
77dc11a658
Author | SHA1 | Date |
---|---|---|
Fred Boniface | 77dc11a658 | |
Fred Boniface | 16fa1c5354 | |
Fred Boniface | dc958ac717 |
|
@ -1,7 +1,7 @@
|
||||||
package background
|
package background
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"math"
|
||||||
"runtime"
|
"runtime"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@ import (
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const frequency = 2 * time.Hour // Figure out a sensible frequency!
|
const frequency = 2 * time.Hour // Figure out a sensible frequency!
|
||||||
|
@ -60,7 +61,8 @@ func debugLog() {
|
||||||
runtime.ReadMemStats(&memStats)
|
runtime.ReadMemStats(&memStats)
|
||||||
|
|
||||||
goroutines := runtime.NumGoroutine()
|
goroutines := runtime.NumGoroutine()
|
||||||
|
heapMem := float64(memStats.HeapAlloc) / (1024 * 1024)
|
||||||
|
heapMemRound := math.Round(heapMem*100) / 100
|
||||||
|
|
||||||
fmt.Printf("\nNumber of goroutines: %d\n", goroutines)
|
log.Msg.Debug("Performance", zap.Int("goroutine-count", goroutines), zap.Float64("heap-mem (MB)", heapMemRound))
|
||||||
fmt.Printf("Heap Allocated memory: %2f MB\n\n", float64(memStats.HeapAlloc)/(1024*1024))
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -68,6 +68,11 @@ func processParsedCif(data *parsedData) error {
|
||||||
|
|
||||||
// Create delete query types and pass to the function which batches the deletions
|
// Create delete query types and pass to the function which batches the deletions
|
||||||
func doDeletions(deletions []*upstreamApi.JsonScheduleV1) error {
|
func doDeletions(deletions []*upstreamApi.JsonScheduleV1) error {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Msg.Panic("Panic:", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
deleteQueries := make([]database.DeleteQuery, 0)
|
deleteQueries := make([]database.DeleteQuery, 0)
|
||||||
for _, item := range deletions {
|
for _, item := range deletions {
|
||||||
query := database.DeleteQuery{
|
query := database.DeleteQuery{
|
||||||
|
|
|
@ -47,6 +47,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||||
log.Msg.Warn("CIF Data updated, but metadata write failed")
|
log.Msg.Warn("CIF Data updated, but metadata write failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
parsed = nil
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -90,6 +91,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata = generateMetadata(&parsed.header)
|
metadata = generateMetadata(&parsed.header)
|
||||||
|
parsed = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
|
ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
|
||||||
|
|
|
@ -74,11 +74,17 @@ func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
|
||||||
|
|
||||||
// Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
|
// Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
|
||||||
func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
log.Msg.Panic("Panic:", zap.Any("panic", r))
|
||||||
|
}
|
||||||
|
}()
|
||||||
// Skip if deletions is empty
|
// Skip if deletions is empty
|
||||||
if len(deletions) == 0 {
|
if len(deletions) == 0 {
|
||||||
log.Msg.Info("No deletions required")
|
log.Msg.Info("No deletions required")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
log.Msg.Info("Running deletions against database", zap.Int("count", len(deletions)))
|
||||||
|
|
||||||
// Prepare deletion tasks
|
// Prepare deletion tasks
|
||||||
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
||||||
|
@ -111,6 +117,7 @@ func CreateCifEntries(schedules []database.Service) error {
|
||||||
log.Msg.Info("No creations required")
|
log.Msg.Info("No creations required")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
log.Msg.Info("Running creations against database", zap.Int("count", len(schedules)))
|
||||||
|
|
||||||
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue