timetable-mgr/corpus/update.go

package corpus

import (
	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
	"git.fjla.uk/owlboard/timetable-mgr/helpers"
	"git.fjla.uk/owlboard/timetable-mgr/log"
	"git.fjla.uk/owlboard/timetable-mgr/nrod"
	"go.uber.org/zap"
)

// Runs all stages of the CORPUS Update process
func RunCorpusUpdate(cfg *helpers.Configuration) error {
	resp, err := nrod.NrodStream(url, cfg)
	if err != nil {
		log.Error("Failed to fetch CORPUS data", zap.Error(err))
		return err
	}

	unsortedCorpusData, err := parseCorpusData(resp)
	if err != nil {
		log.Error("Error parsing Corpus data", zap.Error(err))
		return err
	}

	corpusData := pruneCorpusEntries(*unsortedCorpusData)
	//stationData := createStationEntries(corpusData)

	if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
		log.Warn("CORPUS data may be incomplete")
		log.Error("Error dropping CORPUS Data", zap.Error(err))
		return err
	}
	//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
	//	log.Warn("Stations data may be incomplete")
	//	log.Error("Error dropping stations Data", zap.Error(err))
	//	return err
	//}

	if err := dbAccess.PutManyCorpus(corpusData); err != nil {
		log.Warn("CORPUS data may be incomplete")
		log.Error("Error inserting CORPUS Data", zap.Error(err))
		return err
	}
	//if err := dbAccess.PutManyStations(stationData); err != nil {
	//	log.Warn("Stations data may be incomplete")
	//	log.Error("Error inserting stations data", zap.Error(err))
	//	return err
	//}

	if err := dbAccess.CreateCorpusIndexes(); err != nil {
		log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
	}

	return nil
}