From f33d064d6ee34b4640542923cab4e6f45ef194f4 Mon Sep 17 00:00:00 2001 From: thuanle Date: Mon, 5 Aug 2024 15:06:39 +0700 Subject: [PATCH] tune update db --- .gitignore | 1 + configs/config.go | 5 +-- docker-compose.yml | 7 ++-- go.mod | 1 + go.sum | 2 ++ internal/data/ipdb.go | 81 +++++++++++++++++++++++++++++++++++++++++-- main.go | 6 ++++ pkg/osx/file.go | 34 ++++++++++++++++++ 8 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 pkg/osx/file.go diff --git a/.gitignore b/.gitignore index a4a88b2..8c8849f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.cache/ # If you prefer the allow list template instead of the deny list, see community template: # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore # diff --git a/configs/config.go b/configs/config.go index 546d59a..63f7e64 100644 --- a/configs/config.go +++ b/configs/config.go @@ -6,9 +6,10 @@ var GeoDbSourcePaths = []string{ } const ( - GeoDbFolder = "data/" MmdbmeldConfig = "mmdbmeld.yml" - MmdbDbFile = GeoDbFolder + "geoip-v4.mmdb" + GeoDbFolder = "data/" + MmdbDbFileName = "geoip-v4.mmdb" + MmdbDbFile = GeoDbFolder + MmdbDbFileName ) const ( diff --git a/docker-compose.yml b/docker-compose.yml index 53ca386..edd2f78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,11 @@ services: ip-info: build: . -# ports: -# - "28080:8080" + ports: + - "28080:8080" restart: always volumes: - - ./data:/app/data - network_mode: host + - .cache:/app/.cache environment: - GIN_MODE=release - GIN_TRUSTED_PROXY_IP=127.0.0.1 diff --git a/go.mod b/go.mod index 6623cd5..ea99589 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/oschwald/maxminddb-golang v1.12.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/safing/mmdbmeld v0.3.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect diff --git a/go.sum b/go.sum index 12c4682..13b174a 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/safing/mmdbmeld v0.3.0 h1:eE2SxZF6gD6piH/3nnx1lSlI5kOOgYnoVAQ8NO/RejU= github.com/safing/mmdbmeld v0.3.0/go.mod h1:2tYv3HZcXp26XzcOeirOY9ndlInfYRA9ihJ3x0wlQJw= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/internal/data/ipdb.go b/internal/data/ipdb.go index 0742d5a..a5bbc80 100644 --- a/internal/data/ipdb.go +++ b/internal/data/ipdb.go @@ -1,7 +1,11 @@ package data import ( + "fmt" "github.com/rs/zerolog/log" + "io" + "os" + "path/filepath" "sync" "thuanle.me/ip-info/configs" ) @@ -10,6 +14,8 @@ import reader "github.com/oschwald/maxminddb-golang" type IpDb struct { r *reader.Reader mu sync.RWMutex + + dbFile string } var ( @@ -25,23 +31,92 @@ func Ins() *IpDb { return ins } +func CleanTempFiles() error { + log.Info().Str("dir", configs.GeoDbFolder).Msg("Cleaning temp files") + dir, err := os.Open(configs.GeoDbFolder) + if err != nil { + return err + } + defer dir.Close() + + // List all files in the directory + files, err := dir.Readdir(-1) // -1 means read all files + if err != nil { + return err + } + + // Delete each file + for _, file := range files { + if !file.IsDir() && file.Name() != configs.MmdbDbFileName { + filePath := filepath.Join(configs.GeoDbFolder, file.Name()) + err := os.Remove(filePath) + if err != nil { + return fmt.Errorf("failed to delete file %s: %w", filePath, err) + } + fmt.Printf("Deleted file: %s\n", filePath) + } + } + + return nil +} + func (d *IpDb) Reload() error { d.mu.Lock() defer d.mu.Unlock() - if d.r != nil { - _ = d.r.Close() + wilDeleteFile := d.dbFile + + tmpFile, err := cloneDBFile() + if err != nil { + log.Err(err).Msg("Failed to clone db file") + return err } - r, err := reader.Open(configs.MmdbDbFile) + + d.dbFile = tmpFile + r, err := reader.Open(tmpFile) if err != nil { log.Err(err).Msg("Failed to open mmdb") return err } + tmpR := d.r d.r = r + + if tmpR != nil { + _ = tmpR.Close() + + if wilDeleteFile != "" { + _ = os.Remove(wilDeleteFile) + } + } + return nil } func (d *IpDb) IsLoaded() bool { return d.r != nil } + +func cloneDBFile() (string, error) { + srcFile, err := os.Open(configs.MmdbDbFile) + if err != nil { + return "", err + } + defer srcFile.Close() + + tmpFile, err := os.CreateTemp(configs.GeoDbFolder, "geoip-v4-*.mmdb") + if err != nil { + return "", fmt.Errorf("failed to create temporary file: %v", err) + } + defer tmpFile.Close() + + buf := make([]byte, 1024*1024) // 1 MB buffer + + // Copy the data from srcFile to dstFile + _, err = io.CopyBuffer(tmpFile, srcFile, buf) + if err != nil { + return "", err + } + + return tmpFile.Name(), nil +} diff --git a/main.go b/main.go index 55e3530..f68efe3 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog/log" "os" "os/signal" + "thuanle.me/ip-info/internal/data" "thuanle.me/ip-info/internal/services/api" "thuanle.me/ip-info/internal/services/db_updater" "time" @@ -21,6 +22,11 @@ func main() { log.Err(err).Msg("Error loading .env file") } + err = data.CleanTempFiles() + if err != nil { + log.Err(err).Msg("Failed to clean temp files") + } + stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt) diff --git a/pkg/osx/file.go b/pkg/osx/file.go new file mode 100644 index 0000000..4d34160 --- /dev/null +++ b/pkg/osx/file.go @@ -0,0 +1,34 @@ +package osx + +import ( + "io" + "os" +) + +// Copy copies a file from source to destination +func Copy(from, to string) error { + // Open the source file for reading + srcFile, err := os.Open(from) + if err != nil { + return err + } + defer srcFile.Close() + + // Create the destination file for writing + dstFile, err := os.Create(to) + if err != nil { + return err + } + defer dstFile.Close() + + // Use a buffer to copy the file in chunks + buf := make([]byte, 1024*1024) // 1 MB buffer + + // Copy the data from srcFile to dstFile + _, err = io.CopyBuffer(dstFile, srcFile, buf) + if err != nil { + return err + } + + return nil +}