package cif import ( "io" "time" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/nrod" "go.uber.org/zap" ) // Replaces all existing CIF Data with a new download func runCifFullDownload(cfg *helpers.Configuration) error { preTime := time.Now() log.Info("Downloading all CIF Data") // Download CIF Data file url, err := getUpdateUrl("full") if err != nil { log.Error("Error getting download URL", zap.Error(err)) } dataStream, err := nrod.NrodStream(url, cfg) if err != nil { log.Error("Error downloading CIF data", zap.Error(err)) } // Parse CIF file parsed, err := parseCifDataStream(dataStream) if err != nil { log.Error("Error parsing CIF data", zap.Error(err)) return err } // Try to remove all non-vstp entries, else give up and delete collection count, err := dbAccess.PurgeNonVstp() if err != nil { log.Warn("Error purging non-vstp schedules, dropping collection", zap.Error(err)) dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database. } else { log.Info("Removed non-vstp services", zap.Int64("deletion count", count)) } // Process CIF file err = ProcessParsedCif(parsed) if err != nil { log.Error("Error processing CIF data", zap.Error(err)) } newMeta := generateMetadata(&parsed.Header) ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType) if !ok { log.Warn("CIF Data updated, but metadata write failed") } // Set parsed to nil to encourage garbage collection parsed = nil // Create Indexes err = dbAccess.CreateTimetableIndexes() if err != nil { log.Warn("Error creating timetable indexes, degraded performance", zap.Error(err)) } postTime := time.Now() updateDuration := postTime.Sub(preTime) log.Info("Execution time", zap.Duration("duration", updateDuration)) return nil } // Runs a CIF Update for up to five days func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error { startTime := time.Now() log.Info("Downloading CIF Updates") // Loop over dates for _, time := range days { log.Info("Downloading CIF File", zap.Time("CIF Data from", time)) // Download CIF data file data, err := fetchUpdate(time, cfg) if err != nil { log.Error("Error fetching CIF update", zap.Error(err)) return err } // Parse CIF file parsed, err := parseCifDataStream(data) if err != nil { log.Error("Error parsing CIF data", zap.Error(err)) return err } // Check CIF Metadata log.Debug("Starting metadata checks") reason, update := checkMetadata(metadata, &parsed.Header) if !update { log.Warn("Update file not processed", zap.String("reason", reason)) continue } log.Info("CIF Data is suitable for processing", zap.String("reason", reason)) // Process CIF file err = ProcessParsedCif(parsed) if err != nil { log.Error("Error processing CIF data", zap.Error(err)) } metadata = generateMetadata(&parsed.Header) parsed = nil } // Write metadata to database ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType) if !ok { log.Warn("CIF Data updated, but metadata write failed.") } // Clear out of date schedules cutoff := time.Now().Add(-time.Hour * 24 * 7) log.Debug("Attempting to remove outdated services", zap.Time("scheduleEnd before", cutoff)) count, err := dbAccess.RemoveOutdatedServices(cutoff) if err != nil { log.Warn("Out of date services not removed", zap.Error(err)) } else { log.Info("Out of date services removed", zap.Int64("removal count", count)) } endTime := time.Now() duration := endTime.Sub(startTime) log.Info("CIF Update process ended", zap.Duration("duration", duration)) return nil } // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day. // Note that the previous days update is the latest and is downloaded. func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) { url, err := getUpdateUrl("daily") if err != nil { return nil, err } // Calcuates the day yesterday which is the file that needs downloading updateDay := t.Add(-24 * time.Hour) // Append day string to URL url = url + getDayString(updateDay) log.Debug("Fetching CIF Data", zap.Time("Update_File_Produced", updateDay)) dataStream, err := nrod.NrodStream(url, cfg) if err != nil { return nil, err } return dataStream, nil }