first commit

internal/services/db_updater/downloader.go

@@ -0,0 +1,42 @@
+package db_updater
+
+import (
+	"github.com/go-resty/resty/v2"
+	"github.com/rs/zerolog/log"
+	"net/http"
+	"path"
+	"thuanle.me/ip-info/configs"
+)
+
+var etags = map[string]string{}
+
+func download(url string) bool {
+	log.Info().Str("url", url).Msg("Downloading DB")
+	filename := configs.GeoDbFolder + path.Base(url)
+
+	client := resty.New()
+	resp, err := client.R().
+		SetOutput(filename).
+		SetHeader("If-None-Match", etags[url]).
+		Get(url)
+
+	if err != nil {
+		log.Err(err).
+			Str("url", url).
+			Msg("Failed to fetch DB")
+		return false
+	}
+
+	if resp.StatusCode() == http.StatusOK {
+		etags[url] = resp.Header().Get("Etag")
+	} else {
+		//log.Printf("No update needed: %v", resp.Status())
+		return false
+	}
+
+	log.Info().
+		Str("Etag", etags[url]).
+		Str("filename", filename).
+		Msg("Downloaded DB")
+	return true
+}

internal/services/db_updater/main.go

@@ -0,0 +1,33 @@
+package db_updater
+
+import (
+	"github.com/robfig/cron/v3"
+	"github.com/rs/zerolog/log"
+	"thuanle.me/ip-info/configs"
+	"thuanle.me/ip-info/internal/data"
+)
+
+func StartUpdateDbService() {
+	c := cron.New()
+	_, _ = c.AddFunc("@daily", fetchDbs)
+	c.Start()
+	fetchDbs()
+}
+
+func fetchDbs() {
+	newFlag := false
+	for _, url := range configs.GeoDbSourcePaths {
+		newFlag = newFlag || download(url)
+	}
+
+	if newFlag {
+		log.Info().Msg("New DB downloaded. Recreating mmdb")
+		_ = mergeMmdb()
+
+		err := data.Ins().Reload()
+		if err != nil {
+			log.Err(err).Msg("Failed to reload mmdb")
+			return
+		}
+	}
+}

internal/services/db_updater/merger.go

@@ -0,0 +1,54 @@
+package db_updater
+
+import (
+	"github.com/rs/zerolog/log"
+	"github.com/safing/mmdbmeld"
+	"sync"
+	"thuanle.me/ip-info/configs"
+)
+
+func mergeMmdb() error {
+	c, err := mmdbmeld.LoadConfig(configs.MmdbmeldConfig)
+	if err != nil {
+		log.Err(err).Msg("Failed to load mmdbmeld config")
+		return err
+	}
+
+	for _, db := range c.Databases {
+		log.Info().Str("name", db.Name).Msg("Building mmdb file")
+
+		// Apply defaults.
+		dbP := &db //nolint:gosec,scopelint // Only used within loop.
+		c.Defaults.ApplyTo(dbP)
+
+		// Load sources for database.
+		sources, err := mmdbmeld.LoadSources(db)
+		if err != nil {
+			log.Err(err).Msg("Failed to load sources")
+			return err
+		}
+
+		// Create wait group.
+		var wg sync.WaitGroup
+		wg.Add(1)
+
+		// Start log writer.
+		updates := make(chan string, 100)
+		go func() {
+			defer wg.Done()
+			for msg := range updates {
+				log.Debug().Str("msg", msg).Msg("MMDB update")
+			}
+		}()
+
+		// Read all sources and write to mmdb.
+		err = mmdbmeld.WriteMMDB(db, sources, updates)
+		if err != nil {
+			log.Err(err).Msg("Failed to write mmdb")
+			return err
+		}
+		wg.Wait()
+		log.Info().Str("file", db.Output).Msg("MMDB created")
+	}
+	return nil
+}