timetable-mgr/cif/update.go

package cif

import (
	"errors"
	"io"
	"time"

	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
	"git.fjla.uk/owlboard/timetable-mgr/helpers"
	"git.fjla.uk/owlboard/timetable-mgr/log"
	"git.fjla.uk/owlboard/timetable-mgr/nrod"
	"go.uber.org/zap"
)

// Replaces all existing CIF Data with a new download
func runCifFullDownload(cfg *helpers.Configuration) error {
	log.Msg.Info("Downloading all CIF Data")

	// Download CIF Data file
	url, err := getUpdateUrl("full")
	if err != nil {
		log.Msg.Error("Error getting download URL", zap.Error(err))
	}
	dataStream, err := nrod.NrodStream(url, cfg)
	if err != nil {
		log.Msg.Error("Error downloading CIF data", zap.Error(err))
	}

	// Parse CIF file
	parsed, err := parseCifDataStream(dataStream)
	if err != nil {
		log.Msg.Error("Error parsing CIF data", zap.Error(err))
		return err
	}

	// Drop timetable collection
	dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.

	// Process CIF file
	err = processParsedCif(parsed)
	if err != nil {
		log.Msg.Error("Error processing CIF data", zap.Error(err))
	}

	newMeta := generateMetadata(&parsed.header)
	ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType)
	if !ok {
		log.Msg.Warn("CIF Data updated, but metadata write failed")
	}

	return nil
}

// Runs a CIF Update for up to five days
func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error {
	log.Msg.Info("Downloading CIF Updates")

	// Loop over dates
	for _, time := range days {
		log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time))

		// Download CIF data file
		data, err := fetchUpdate(time, cfg)
		if err != nil {
			log.Msg.Error("Error fetching CIF update", zap.Error(err))
			return err
		}

		// Parse CIF file
		parsed, err := parseCifDataStream(data)
		if err != nil {
			log.Msg.Error("Error parsing CIF data", zap.Error(err))
			return err
		}

		log.Msg.Debug("Starting metadata checks")

		// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
		// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
		//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
		// Somehow skip the below checks!  Maybe I need a checkMetadata function!
		//}

		// Check CIF Sequence
		// Skip if LastSequence is >= to this sequence
		if metadata.LastSequence >= parsed.header.Metadata.Sequence {
			log.Msg.Info("Skipping CIF file, already processed", zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New sequence", parsed.header.Metadata.Sequence))
			continue
		}

		// Fail if sequence number is not as expected
		if parsed.header.Metadata.Sequence != metadata.LastSequence+1 {
			err := errors.New("out of sequence CIF data")
			log.Msg.Error("CIF sequence not as expected", zap.Error(err), zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New Sequence", parsed.header.Metadata.Sequence))
			return err
		}
		log.Msg.Debug("Metadata checks complete")

		// Do further sequence checks - parsed.header.Metadata.Sequence MUST = metadata.LastSequence + 1

		log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
		// Process CIF file

		// Temporarily disable METADATA GENERATION AND WRITING
		//		metadata = generateMetadata(&parsed.header)
	}

	ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
	if !ok {
		log.Msg.Warn("CIF Data updated, but metadata write failed.")
	}

	return nil
}

// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
	url, err := getUpdateUrl("daily")
	if err != nil {
		return nil, err
	}

	// Append day string to URL
	url = url + getDayString(t)

	dataStream, err := nrod.NrodStream(url, cfg)
	if err != nil {
		return nil, err
	}
	return dataStream, nil
}