From 8f0e9759c42bc46a2c0c0256c99e1e5487cb2e00 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Mon, 21 Oct 2024 21:04:37 +0100 Subject: [PATCH] Add PIS Logic --- go.mod | 1 + go.sum | 1 + pis/data.go | 29 +++++++++ pis/types.go | 6 ++ pis/update.go | 161 +++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 pis/data.go diff --git a/go.mod b/go.mod index c5439bd..6831ad3 100644 --- a/go.mod +++ b/go.mod @@ -23,4 +23,5 @@ require ( golang.org/x/crypto v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/text v0.16.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8a7d1bf..50f26f9 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pis/data.go b/pis/data.go new file mode 100644 index 0000000..483a843 --- /dev/null +++ b/pis/data.go @@ -0,0 +1,29 @@ +package pis + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// Process the YAML data to a struct +func processYaml(yamlStr string) (*PisData, error) { + var pis PisData + + err := yaml.Unmarshal([]byte(yamlStr), &pis) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML: %v", err) + } + + err = deduplicateCodes(&pis) + if err != nil { + return nil, err + } + + return &pis, nil +} + +// Deduplicate data in place and return error if failed +func deduplicateCodes(pis *PisData) error { + return fmt.Errorf("deduplication logic not present, unable to update") +} diff --git a/pis/types.go b/pis/types.go index 1a4da2f..2f57b4f 100644 --- a/pis/types.go +++ b/pis/types.go @@ -7,3 +7,9 @@ type GiteaReleaseData struct { Draft bool `json:"draft"` Prerelease bool `json:"prerelease"` } + +type PisData struct { + Code string `json:"code"` + Stops []string `json:"stops"` + Operator string `json:"operator"` +} diff --git a/pis/update.go b/pis/update.go index 2f7b436..1ec6a32 100644 --- a/pis/update.go +++ b/pis/update.go @@ -1,11 +1,166 @@ package pis +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +const ( + destPath = "/tmp/pis.tar.gz" + extractPath = "/tmp/extract" +) + // Downloads the release tarball, extracts then applies to database func runUpdate(tarballUrl string) error { - // Download and extract tarball - // load PIS yaml files - // Run code deduplication + err := downloadFile(tarballUrl, destPath) + if err != nil { + return err + } + + // Extract to disk + file, err := os.Open(destPath) + if err != nil { + fmt.Printf("Error opening file: %v\n", err) + return err + } + defer file.Close() + + if err := extractFiles(file, extractPath); err != nil { + fmt.Printf("Error extracting file: %v\n", err) + return err + } + + // Load YAML to string + pisData, err := extractYamlData(extractPath) + if err != nil { + return err + } + // Replace database collection with new data // Ensure indeces are present + + // Cleanup files + cleanupFiles(destPath, extractPath) return nil } + +// Download the tarball to disk +func downloadFile(url, filepath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + + defer resp.Body.Close() + + out, err := os.Create(filepath) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// Extract tarball to disk +func extractFiles(gzipStream io.Reader, dest string) error { + uncompressedStream, err := gzip.NewReader(gzipStream) + if err != nil { + return err + } + + defer uncompressedStream.Close() + + tarReader := tar.NewReader(uncompressedStream) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + filePath := filepath.Join(dest, header.Name) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(filepath.Join(dest, header.Name), 0755); err != nil { + return err + } + case tar.TypeReg: + outFile, err := os.Create(filePath) + if err != nil { + return err + } + + if _, err := io.Copy(outFile, tarReader); err != nil { + return err + } + outFile.Close() + default: + fmt.Printf("Unable to handle filetype %c in %s\n", header.Typeflag, header.Name) + } + } + return nil +} + +// Return YAML PIS files as a string +func extractYamlData(dir string) (string, error) { + var allContent strings.Builder // Using a string builder to accumulate content + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err // Only returning error since Walk callback doesn't accept string + } + + // Check if the path contains 'pis' and has a .yaml or .yml extension + if strings.Contains(path, "/pis/") && !info.IsDir() && + (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) { + fmt.Printf("Processing YAML file in 'pis' directory: %s\n", path) + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open YAML file %s: %v", path, err) + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return fmt.Errorf("failed to read YAML file %s: %v", path, err) + } + + // Accumulate content from each YAML file in 'pis' directory + allContent.Write(content) + allContent.WriteString("\n") // Add a newline between file contents + } + + return nil + }) + + if err != nil { + return "", err + } + + // Return the accumulated content as a single string + return allContent.String(), nil +} + +// Cleans up downloaded and extracted files +func cleanupFiles(paths ...string) { + for _, path := range paths { + err := os.RemoveAll(path) + if err != nil { + fmt.Printf("Error removing %s: %v\n", path, err) + } + } +}