package cif import ( "errors" "io" "time" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/nrod" "go.uber.org/zap" ) // Replaces all existing CIF Data with a new download func runCifFullDownload(cfg *helpers.Configuration) error { log.Msg.Info("Downloading all CIF Data") // Download CIF Data file url, err := getUpdateUrl("full") if err != nil { log.Msg.Error("Error getting download URL", zap.Error(err)) } dataStream, err := nrod.NrodStream(url, cfg) if err != nil { log.Msg.Error("Error downloading CIF data", zap.Error(err)) } // DOES NOT WORK WITH NEW DOWNLOAD STREAMING // If debug mode is on, call debugWriteDownload // if helpers.Runtime == "debug" { // debugWriteDownload(dataStream) // } // Parse CIF file parsed, err := parseCifDataStream(dataStream) if err != nil { log.Msg.Error("Error parsing CIF data", zap.Error(err)) return err } // Look to stream data onwards to the parsing function // Make `data` a nil pointer as it is no longer required dataStream = nil // Drop timetable collection dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database. // Process CIF file err = processParsedCif(parsed) if err != nil { log.Msg.Error("Error processing CIF data", zap.Error(err)) } newMeta := generateMetadata(&parsed.header) ok := dbAccess.PutCifMetadata(newMeta) if !ok { log.Msg.Warn("CIF Data updated, but metadata write failed") } return nil } // Runs a CIF Update for up to five days func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error { log.Msg.Info("Downloading CIF Updates") // Loop over dates for _, time := range days { log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time)) // Download CIF data file data, err := fetchUpdate(time, cfg) if err != nil { log.Msg.Error("Error fetching CIF update", zap.Error(err)) return err } // DOES NOT WORK WITH NEW NROD STREAMER // If debug mode is on, call debugWriteDownload // if helpers.Runtime == "debug" { // debugWriteDownload(data) // } // Parse CIF file parsed, err := parseCifDataStream(data) if err != nil { log.Msg.Error("Error parsing CIF data", zap.Error(err)) return err } // Make `data` a nil pointer as it is no longer required data = nil log.Msg.Debug("Starting metadata checks") // Check CIF Sequence // Skip if LastSequence is >= to this sequence if metadata.LastSequence >= parsed.header.Metadata.Sequence { log.Msg.Info("Skipping CIF file, already processed", zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New sequence", parsed.header.Metadata.Sequence)) continue } // Fail if sequence number is not as expected if parsed.header.Metadata.Sequence != metadata.LastSequence+1 { err := errors.New("out of sequence CIF data") log.Msg.Error("CIF sequence not as expected", zap.Error(err), zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New Sequence", parsed.header.Metadata.Sequence)) return err } log.Msg.Debug("Metadata checks complete") // Do further sequence checks - parsed.header.Metadata.Sequence MUST = metadata.LastSequence + 1 log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<") // Process CIF file // Temporarily disable METADATA GENERATION AND WRITING // metadata = generateMetadata(&parsed.header) } ok := dbAccess.PutCifMetadata(metadata) if !ok { log.Msg.Warn("CIF Data updated, but metadata write failed.") } return nil } // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) { url, err := getUpdateUrl("daily") if err != nil { return nil, err } // Append day string to URL url = url + getDayString(t) dataStream, err := nrod.NrodStream(url, cfg) if err != nil { return nil, err } return dataStream, nil }