package pis import ( "archive/tar" "compress/gzip" "fmt" "io" "net/http" "os" "path/filepath" "strings" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/log" "go.uber.org/zap" ) const ( destPath = "/tmp/pis.tar.gz" extractPath = "/tmp/extract" ) // Downloads the release tarball, extracts then applies to database func runUpdate(tarballUrl string) error { err := downloadFile(tarballUrl, destPath) if err != nil { return err } // Extract to disk file, err := os.Open(destPath) if err != nil { return err } defer file.Close() if err := extractFiles(file, extractPath); err != nil { return err } // Load YAML to string pisData, err := extractYamlData(extractPath) if err != nil { return err } pisSlice, err := processYaml(pisData) if err != nil { return err } fmt.Println(&pisSlice) err = dbAccess.DropCollection(dbAccess.PisCollection) if err != nil { return err } count, err := dbAccess.PutPisData(pisSlice) if err != nil { return err } log.Info("Insterted new PIS Data", zap.Int64("PIS Codes", count)) err = dbAccess.CreatePisIndeces() if err != nil { log.Error("Failed to create PIS Indeces, poor performance expected", zap.Error(err)) } // Cleanup files cleanupFiles(destPath, extractPath) return nil } // Download the tarball to disk func downloadFile(url, filepath string) error { resp, err := http.Get(url) if err != nil { return err } defer resp.Body.Close() out, err := os.Create(filepath) if err != nil { return err } defer out.Close() _, err = io.Copy(out, resp.Body) return err } // Extract tarball to disk func extractFiles(gzipStream io.Reader, dest string) error { uncompressedStream, err := gzip.NewReader(gzipStream) if err != nil { return err } defer uncompressedStream.Close() tarReader := tar.NewReader(uncompressedStream) for { header, err := tarReader.Next() if err == io.EOF { break } if err != nil { return err } filePath := filepath.Join(dest, header.Name) switch header.Typeflag { case tar.TypeDir: if err := os.MkdirAll(filepath.Join(dest, header.Name), 0755); err != nil { return err } case tar.TypeReg: outFile, err := os.Create(filePath) if err != nil { return err } if _, err := io.Copy(outFile, tarReader); err != nil { return err } outFile.Close() default: fmt.Printf("Unable to handle filetype %c in %s\n", header.Typeflag, header.Name) } } return nil } // Return YAML PIS files as a string func extractYamlData(dir string) (string, error) { var allContent strings.Builder // Using a string builder to accumulate content err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err // Only returning error since Walk callback doesn't accept string } // Check if the path contains 'pis' and has a .yaml or .yml extension if strings.Contains(path, "/pis/") && !info.IsDir() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) { fmt.Printf("Processing YAML file in 'pis' directory: %s\n", path) file, err := os.Open(path) if err != nil { return fmt.Errorf("failed to open YAML file %s: %v", path, err) } defer file.Close() content, err := io.ReadAll(file) if err != nil { return fmt.Errorf("failed to read YAML file %s: %v", path, err) } // Accumulate content from each YAML file in 'pis' directory allContent.Write(content) allContent.WriteString("\n") // Add a newline between file contents } return nil }) if err != nil { return "", err } // Return the accumulated content as a single string return allContent.String(), nil } // Cleans up downloaded and extracted files func cleanupFiles(paths ...string) { for _, path := range paths { err := os.RemoveAll(path) if err != nil { fmt.Printf("Error removing %s: %v\n", path, err) } } }