Set unordered for bulk CIF deletions from DB & reorganise dbAccess module.

dbAccess/access.go

@@ -2,79 +2,16 @@ package dbAccess
 
 import (
 	"context"
-	"time"
 
 	"git.fjla.uk/owlboard/go-types/pkg/database"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"go.uber.org/zap"
 )
 
 const timetableCollection string = "timetable"
 
-// CAUTION: Drops the collection named in collectionName
-func DropCollection(collectionName string) error {
-	log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
-	database := MongoClient.Database(databaseName)
-	collection := database.Collection(collectionName)
-
-	err := collection.Drop(context.Background())
-	if err != nil {
-		log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
-func CheckUpdateTime(collectionName string) (int64, error) {
-	database := MongoClient.Database(databaseName)
-	collection := database.Collection(metaCollection)
-	filter := bson.D{
-		{Key: "target", Value: collectionName},
-		{Key: "type", Value: "collection"},
-	}
-	var result struct {
-		Updated int64 `bson:"updated"`
-	}
-	err := collection.FindOne(context.Background(), filter).Decode(&result)
-	if err != nil {
-		return 0, err
-	}
-	return result.Updated, nil
-}
-
-// Sets a new update time (unix timestamp) of the collection named in collectionName.  The update time is calculated within the function.
-func SetUpdateTime(collectionName string) error {
-	log.Msg.Info("Setting update time", zap.String("collection", collectionName))
-	database := MongoClient.Database(databaseName)
-	collection := database.Collection("meta")
-	options := options.Update().SetUpsert(true)
-	updateTime := time.Now().Unix()
-	filter := bson.M{
-		"target": collectionName,
-		"type":   "collection",
-	}
-	update := bson.M{
-		"$set": bson.M{
-			"updated":      updateTime,
-			"updated_time": time.Now().In(time.UTC),
-			"target":       collectionName,
-			"type":         "collection",
-		},
-	}
-	_, err := collection.UpdateOne(context.Background(), filter, update, options)
-
-	if err != nil {
-		log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
-		return err
-	}
-	return nil
-}
-
 // Pushes the current version of this application to the data base 'versions' collection.
 // Currently uses the old name of mq-client
 func PushVersionToDb() {

dbAccess/cif.go

@@ -93,7 +93,9 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error {
 		bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter))
 	}
 
-	_, err := collection.BulkWrite(context.Background(), bulkDeletions)
+	bulkWriteOptions := options.BulkWrite().SetOrdered(false)
+
+	_, err := collection.BulkWrite(context.Background(), bulkDeletions, bulkWriteOptions)
 	if err != nil {
 		log.Msg.Error("Error deleting documents", zap.Error(err))
 		return err

dbAccess/common.go

@@ -1 +1,72 @@
 package dbAccess
+
+import (
+	"context"
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.uber.org/zap"
+)
+
+// CAUTION: Drops the collection named in collectionName
+func DropCollection(collectionName string) error {
+	log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection(collectionName)
+
+	err := collection.Drop(context.Background())
+	if err != nil {
+		log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
+func CheckUpdateTime(collectionName string) (int64, error) {
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection(metaCollection)
+	filter := bson.D{
+		{Key: "target", Value: collectionName},
+		{Key: "type", Value: "collection"},
+	}
+	var result struct {
+		Updated int64 `bson:"updated"`
+	}
+	err := collection.FindOne(context.Background(), filter).Decode(&result)
+	if err != nil {
+		return 0, err
+	}
+	return result.Updated, nil
+}
+
+// Sets a new update time (unix timestamp) of the collection named in collectionName.  The update time is calculated within the function.
+func SetUpdateTime(collectionName string) error {
+	log.Msg.Info("Setting update time", zap.String("collection", collectionName))
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection("meta")
+	options := options.Update().SetUpsert(true)
+	updateTime := time.Now().Unix()
+	filter := bson.M{
+		"target": collectionName,
+		"type":   "collection",
+	}
+	update := bson.M{
+		"$set": bson.M{
+			"updated":      updateTime,
+			"updated_time": time.Now().In(time.UTC),
+			"target":       collectionName,
+			"type":         "collection",
+		},
+	}
+	_, err := collection.UpdateOne(context.Background(), filter, update, options)
+
+	if err != nil {
+		log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
+		return err
+	}
+	return nil
+}

dbAccess/contants.go

@@ -5,4 +5,3 @@ const CorpusCollection string = "corpus"
 const StationsCollection string = "stations"
 const metaCollection string = "meta"
 const TimetableCollection string = "timetable"
-const batchsize int = 100

dbAccess/types.go

@@ -0,0 +1,6 @@
+package dbAccess
+
+// This file should define types used within dbAccess.
+// Any types representing database or upstream API resources should
+// instead be defined in git.fjla.uk/owlboard/go-types and also be
+// reflected in git.fjla.uk/owlboard/ts-types