Add additional logging to identify why CIF updates were sometimes failing

background/ticker.go

@@ -1,7 +1,7 @@
 package background
 
 import (
-	"fmt"
+	"math"
 	"runtime"
 	"time"
 
@@ -9,6 +9,7 @@ import (
 	"git.fjla.uk/owlboard/timetable-mgr/corpus"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
 )
 
 const frequency = 2 * time.Hour // Figure out a sensible frequency!
@@ -60,7 +61,8 @@ func debugLog() {
 	runtime.ReadMemStats(&memStats)
 
 	goroutines := runtime.NumGoroutine()
+	heapMem := float64(memStats.HeapAlloc) / (1024 * 1024)
+	heapMemRound := math.Round(heapMem*100) / 100
 
-	fmt.Printf("\nNumber of goroutines: %d\n", goroutines)
+	log.Msg.Debug("Performance", zap.Int("goroutine-count", goroutines), zap.Float64("heap-mem (MB)", heapMemRound))
-	fmt.Printf("Heap Allocated memory: %2f MB\n\n", float64(memStats.HeapAlloc)/(1024*1024))
 }

cif/process.go

@@ -68,6 +68,11 @@ func processParsedCif(data *parsedData) error {
 
 // Create delete query types and pass to the function which batches the deletions
 func doDeletions(deletions []*upstreamApi.JsonScheduleV1) error {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Msg.Panic("Panic:", zap.Any("panic", r))
+		}
+	}()
 	deleteQueries := make([]database.DeleteQuery, 0)
 	for _, item := range deletions {
 		query := database.DeleteQuery{

cif/update.go

@@ -47,6 +47,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 		log.Msg.Warn("CIF Data updated, but metadata write failed")
 	}
 
+	parsed = nil
 	return nil
 }
 
@@ -90,6 +91,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 		}
 
 		metadata = generateMetadata(&parsed.header)
+		parsed = nil
 	}
 
 	ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)

dbAccess/cif.go

@@ -74,11 +74,17 @@ func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
 
 // Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
 func DeleteCifEntries(deletions []database.DeleteQuery) error {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Msg.Panic("Panic:", zap.Any("panic", r))
+		}
+	}()
 	// Skip if deletions is empty
 	if len(deletions) == 0 {
 		log.Msg.Info("No deletions required")
 		return nil
 	}
+	log.Msg.Info("Running deletions against database", zap.Int("count", len(deletions)))
 
 	// Prepare deletion tasks
 	collection := MongoClient.Database(databaseName).Collection(timetableCollection)
@@ -111,6 +117,7 @@ func CreateCifEntries(schedules []database.Service) error {
 		log.Msg.Info("No creations required")
 		return nil
 	}
+	log.Msg.Info("Running creations against database", zap.Int("count", len(schedules)))
 
 	collection := MongoClient.Database(databaseName).Collection(timetableCollection)