From 251a1905586badd15565aee74ce740d67c0b52f7 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Thu, 11 Apr 2024 21:06:10 +0100 Subject: [PATCH] Set unordered for bulk CIF deletions from DB & reorganise dbAccess module. --- dbAccess/access.go | 63 --------------------------------------- dbAccess/cif.go | 4 ++- dbAccess/common.go | 71 ++++++++++++++++++++++++++++++++++++++++++++ dbAccess/contants.go | 1 - dbAccess/types.go | 6 ++++ 5 files changed, 80 insertions(+), 65 deletions(-) create mode 100644 dbAccess/types.go diff --git a/dbAccess/access.go b/dbAccess/access.go index 6e8e847..952909c 100644 --- a/dbAccess/access.go +++ b/dbAccess/access.go @@ -2,79 +2,16 @@ package dbAccess import ( "context" - "time" "git.fjla.uk/owlboard/go-types/pkg/database" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/options" - "go.uber.org/zap" ) const timetableCollection string = "timetable" -// CAUTION: Drops the collection named in collectionName -func DropCollection(collectionName string) error { - log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName)) - database := MongoClient.Database(databaseName) - collection := database.Collection(collectionName) - - err := collection.Drop(context.Background()) - if err != nil { - log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err)) - return err - } - - return nil -} - -// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection -func CheckUpdateTime(collectionName string) (int64, error) { - database := MongoClient.Database(databaseName) - collection := database.Collection(metaCollection) - filter := bson.D{ - {Key: "target", Value: collectionName}, - {Key: "type", Value: "collection"}, - } - var result struct { - Updated int64 `bson:"updated"` - } - err := collection.FindOne(context.Background(), filter).Decode(&result) - if err != nil { - return 0, err - } - return result.Updated, nil -} - -// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function. -func SetUpdateTime(collectionName string) error { - log.Msg.Info("Setting update time", zap.String("collection", collectionName)) - database := MongoClient.Database(databaseName) - collection := database.Collection("meta") - options := options.Update().SetUpsert(true) - updateTime := time.Now().Unix() - filter := bson.M{ - "target": collectionName, - "type": "collection", - } - update := bson.M{ - "$set": bson.M{ - "updated": updateTime, - "updated_time": time.Now().In(time.UTC), - "target": collectionName, - "type": "collection", - }, - } - _, err := collection.UpdateOne(context.Background(), filter, update, options) - - if err != nil { - log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err)) - return err - } - return nil -} - // Pushes the current version of this application to the data base 'versions' collection. // Currently uses the old name of mq-client func PushVersionToDb() { diff --git a/dbAccess/cif.go b/dbAccess/cif.go index d10028b..fafb0c2 100644 --- a/dbAccess/cif.go +++ b/dbAccess/cif.go @@ -93,7 +93,9 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error { bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter)) } - _, err := collection.BulkWrite(context.Background(), bulkDeletions) + bulkWriteOptions := options.BulkWrite().SetOrdered(false) + + _, err := collection.BulkWrite(context.Background(), bulkDeletions, bulkWriteOptions) if err != nil { log.Msg.Error("Error deleting documents", zap.Error(err)) return err diff --git a/dbAccess/common.go b/dbAccess/common.go index bb922db..7d9a8e4 100644 --- a/dbAccess/common.go +++ b/dbAccess/common.go @@ -1 +1,72 @@ package dbAccess + +import ( + "context" + "time" + + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" +) + +// CAUTION: Drops the collection named in collectionName +func DropCollection(collectionName string) error { + log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName)) + database := MongoClient.Database(databaseName) + collection := database.Collection(collectionName) + + err := collection.Drop(context.Background()) + if err != nil { + log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err)) + return err + } + + return nil +} + +// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection +func CheckUpdateTime(collectionName string) (int64, error) { + database := MongoClient.Database(databaseName) + collection := database.Collection(metaCollection) + filter := bson.D{ + {Key: "target", Value: collectionName}, + {Key: "type", Value: "collection"}, + } + var result struct { + Updated int64 `bson:"updated"` + } + err := collection.FindOne(context.Background(), filter).Decode(&result) + if err != nil { + return 0, err + } + return result.Updated, nil +} + +// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function. +func SetUpdateTime(collectionName string) error { + log.Msg.Info("Setting update time", zap.String("collection", collectionName)) + database := MongoClient.Database(databaseName) + collection := database.Collection("meta") + options := options.Update().SetUpsert(true) + updateTime := time.Now().Unix() + filter := bson.M{ + "target": collectionName, + "type": "collection", + } + update := bson.M{ + "$set": bson.M{ + "updated": updateTime, + "updated_time": time.Now().In(time.UTC), + "target": collectionName, + "type": "collection", + }, + } + _, err := collection.UpdateOne(context.Background(), filter, update, options) + + if err != nil { + log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err)) + return err + } + return nil +} diff --git a/dbAccess/contants.go b/dbAccess/contants.go index 29c11bb..612f334 100644 --- a/dbAccess/contants.go +++ b/dbAccess/contants.go @@ -5,4 +5,3 @@ const CorpusCollection string = "corpus" const StationsCollection string = "stations" const metaCollection string = "meta" const TimetableCollection string = "timetable" -const batchsize int = 100 diff --git a/dbAccess/types.go b/dbAccess/types.go new file mode 100644 index 0000000..6404fca --- /dev/null +++ b/dbAccess/types.go @@ -0,0 +1,6 @@ +package dbAccess + +// This file should define types used within dbAccess. +// Any types representing database or upstream API resources should +// instead be defined in git.fjla.uk/owlboard/go-types and also be +// reflected in git.fjla.uk/owlboard/ts-types