From 55521ab750429c1f659e61c94c072056dc2d0cf0 Mon Sep 17 00:00:00 2001 From: AiAe Date: Thu, 24 Sep 2026 23:42:38 +0300 Subject: [PATCH 1/4] Add cache prune cron/command --- cmd/console/cmd.go | 1 + cmd/console/commands/cache_prune.go | 41 +++++++++++++ cmd/cron/main.go | 1 + config.example.json | 7 ++- config/config.go | 1 + files/cache.go | 28 ++++++++- files/cache_prune.go | 95 +++++++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 cmd/console/commands/cache_prune.go create mode 100644 files/cache_prune.go diff --git a/cmd/console/cmd.go b/cmd/console/cmd.go index 3876fa0..a4f7480 100644 --- a/cmd/console/cmd.go +++ b/cmd/console/cmd.go @@ -38,6 +38,7 @@ func init() { // Commands RootCmd.AddCommand(commands.CacheClearCmd) + RootCmd.AddCommand(commands.CachePruneCmd) RootCmd.AddCommand(commands.CacheLeaderboardCmd) RootCmd.AddCommand(commands.CacheClanLeaderboard) RootCmd.AddCommand(commands.ElasticIndexMapsets) diff --git a/cmd/console/commands/cache_prune.go b/cmd/console/commands/cache_prune.go new file mode 100644 index 0000000..f186d11 --- /dev/null +++ b/cmd/console/commands/cache_prune.go @@ -0,0 +1,41 @@ +package commands + +import ( + "github.com/Quaver/api2/config" + "github.com/Quaver/api2/files" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var CachePruneCmd = &cobra.Command{ + Use: "cache:prune", + Short: "Finds cached files unused for 90 days", + RunE: func(cmd *cobra.Command, args []string) error { + deleteFiles, err := cmd.Flags().GetBool("delete") + if err != nil { + return err + } + return RunCachePrune(deleteFiles) + }, +} + +func init() { + CachePruneCmd.Flags().Bool("delete", false, "delete old cached files instead of performing a dry run") +} + +func RunCachePrune(deleteFiles bool) error { + logrus.Infof("Cache prune scanning %s", config.Instance.Cache.DataDirectory) + result, err := files.PruneCache(deleteFiles) + megabytes := result.Bytes / 1_000_000 + + if deleteFiles { + logrus.Infof("Cache prune deleted %d files (%d MB)", result.Files, megabytes) + } else { + logrus.Infof("Cache prune dry run found %d files (%d MB)", result.Files, megabytes) + } + + if err != nil { + logrus.Errorf("Error pruning cached files: %v", err) + } + return err +} diff --git a/cmd/cron/main.go b/cmd/cron/main.go index 4a25dc8..159743e 100644 --- a/cmd/cron/main.go +++ b/cmd/cron/main.go @@ -46,6 +46,7 @@ func main() { registerCronJob(c, jobs.RankClanMap.Job, func() { commands.ClanRankMapCmd.Run(nil, nil) }) registerCronJob(c, jobs.DenyOnHoldOneMonth.Job, func() { commands.DenyOnHoldCmd.Run(nil, nil) }) registerCronJob(c, jobs.ClanRecalculate.Job, func() { commands.ClanRecalculateCommand.Run(nil, nil) }) + registerCronJob(c, jobs.CachePrune.Job, func() { commands.RunCachePrune(true) }) c.Start() diff --git a/config.example.json b/config.example.json index a167a0c..cc7c3e6 100644 --- a/config.example.json +++ b/config.example.json @@ -138,6 +138,11 @@ "enabled": true, "name": "Performs a full recalculation on clans", "schedule": "0 * * * *" + }, + "cache_prune": { + "enabled": true, + "name": "Deletes 90+ days older cached maps, mapsets, and replays", + "schedule": "0 3 * * *" } } -} \ No newline at end of file +} diff --git a/config/config.go b/config/config.go index 16645c6..f20a57c 100644 --- a/config/config.go +++ b/config/config.go @@ -119,6 +119,7 @@ type Config struct { RankClanMap CronJob `json:"rank_clan_map"` DenyOnHoldOneMonth CronJob `json:"deny_on_hold_one_month"` ClanRecalculate CronJob `json:"clan_recalculate"` + CachePrune CronJob `json:"cache_prune"` } `json:"cron"` } diff --git a/files/cache.go b/files/cache.go index d972459..1f2c876 100644 --- a/files/cache.go +++ b/files/cache.go @@ -10,8 +10,11 @@ import ( "github.com/sirupsen/logrus" "os" "path/filepath" + "time" ) +const cacheAccessUpdateInterval = 24 * time.Hour + // CreateDirectories Creates the directories needed for the cache func CreateDirectories() { if err := os.RemoveAll(GetTempDirectory()); err != nil { @@ -43,7 +46,7 @@ func CacheQuaFile(mapQua *db.MapQua) (string, error) { path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getMapsDirectory(), fileName)) // Check for existing file & see if md5 hash matches - if _, err := os.Stat(path); err == nil { + if info, err := os.Stat(path); err == nil { md5, err := GetFileMD5(path) if err != nil { @@ -51,6 +54,9 @@ func CacheQuaFile(mapQua *db.MapQua) (string, error) { } if md5 == mapQua.MD5 { + if err := markCacheAccess(path, info); err != nil { + logrus.Warnf("Failed to update access time for %v: %v", path, err) + } return path, nil } } @@ -89,7 +95,7 @@ func CacheMapset(mapset *db.Mapset) (string, error) { path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getMapsetDirectory(), fileName)) // Check MD5 hash of existing file - if _, err := os.Stat(path); err == nil { + if info, err := os.Stat(path); err == nil { md5, err := GetFileMD5(path) if err != nil { @@ -97,6 +103,9 @@ func CacheMapset(mapset *db.Mapset) (string, error) { } if md5 == mapset.PackageMD5 { + if err := markCacheAccess(path, info); err != nil { + logrus.Warnf("Failed to update access time for %v: %v", path, err) + } return path, nil } } @@ -113,7 +122,10 @@ func CacheReplay(scoreId int) (string, error) { fileName := fmt.Sprintf("%v.qr", scoreId) path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getReplayDirectory(), fileName)) - if _, err := os.Stat(path); err == nil { + if info, err := os.Stat(path); err == nil { + if err := markCacheAccess(path, info); err != nil { + logrus.Warnf("Failed to update access time for %v: %v", path, err) + } return path, nil } @@ -144,3 +156,13 @@ func GetBackupsDirectory() string { func GetTempDirectory() string { return "../../temp" } + +func markCacheAccess(path string, info os.FileInfo) error { + now := time.Now() + + if now.Sub(info.ModTime()) < cacheAccessUpdateInterval { + return nil + } + + return os.Chtimes(path, now, now) +} diff --git a/files/cache_prune.go b/files/cache_prune.go new file mode 100644 index 0000000..1c70cf6 --- /dev/null +++ b/files/cache_prune.go @@ -0,0 +1,95 @@ +package files + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Quaver/api2/config" +) + +const cacheRetention = 90 * 24 * time.Hour + +type CachePruneResult struct { + Files int + Bytes int64 +} + +// PruneCache finds cached maps, mapsets, and replays that are unused for 90 days. +func PruneCache(deleteFiles bool) (CachePruneResult, error) { + var result CachePruneResult + root := config.Instance.Cache.DataDirectory + cutoff := time.Now().Add(-cacheRetention) + cacheDirectories := []struct { + name string + extension string + }{ + {"maps", ".qua"}, + {"mapsets", ".qp"}, + {"replays", ".qr"}, + } + + for _, cacheDirectory := range cacheDirectories { + directory := filepath.Join(root, cacheDirectory.name) + entries, err := os.ReadDir(directory) + if err != nil { + return result, fmt.Errorf("read cache directory %s: %w", directory, err) + } + + for _, entry := range entries { + if !isCacheFile(entry.Name(), cacheDirectory.extension) { + continue + } + + path := filepath.Join(directory, entry.Name()) + info, err := entry.Info() + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return result, fmt.Errorf("inspect cache file %s: %w", path, err) + } + if !info.Mode().IsRegular() || !info.ModTime().Before(cutoff) { + continue + } + + if deleteFiles { + // Skip files that were used or replaced while scanning. + currentInfo, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return result, fmt.Errorf("inspect cache file %s: %w", path, err) + } + if !os.SameFile(info, currentInfo) || !currentInfo.ModTime().Before(cutoff) { + continue + } + if err := os.Remove(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + return result, fmt.Errorf("remove cache file %s: %w", path, err) + } + } + + result.Files++ + result.Bytes += info.Size() + } + } + + return result, nil +} + +func isCacheFile(name string, extension string) bool { + if !strings.HasSuffix(name, extension) { + return false + } + + id, err := strconv.Atoi(strings.TrimSuffix(name, extension)) + return err == nil && id > 0 +} From d6c6881c80f6eb7ef0dd636bec5fe79463d75e0e Mon Sep 17 00:00:00 2001 From: AiAe Date: Fri, 25 Sep 2026 09:42:58 +0300 Subject: [PATCH 2/4] Simplify the loop further --- files/cache_prune.go | 44 ++------------------------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/files/cache_prune.go b/files/cache_prune.go index 1c70cf6..5663b9c 100644 --- a/files/cache_prune.go +++ b/files/cache_prune.go @@ -5,8 +5,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" - "strings" "time" "github.com/Quaver/api2/config" @@ -24,32 +22,17 @@ func PruneCache(deleteFiles bool) (CachePruneResult, error) { var result CachePruneResult root := config.Instance.Cache.DataDirectory cutoff := time.Now().Add(-cacheRetention) - cacheDirectories := []struct { - name string - extension string - }{ - {"maps", ".qua"}, - {"mapsets", ".qp"}, - {"replays", ".qr"}, - } - for _, cacheDirectory := range cacheDirectories { - directory := filepath.Join(root, cacheDirectory.name) + for _, name := range []string{"maps", "mapsets", "replays"} { + directory := filepath.Join(root, name) entries, err := os.ReadDir(directory) if err != nil { return result, fmt.Errorf("read cache directory %s: %w", directory, err) } for _, entry := range entries { - if !isCacheFile(entry.Name(), cacheDirectory.extension) { - continue - } - path := filepath.Join(directory, entry.Name()) info, err := entry.Info() - if errors.Is(err, os.ErrNotExist) { - continue - } if err != nil { return result, fmt.Errorf("inspect cache file %s: %w", path, err) } @@ -58,21 +41,7 @@ func PruneCache(deleteFiles bool) (CachePruneResult, error) { } if deleteFiles { - // Skip files that were used or replaced while scanning. - currentInfo, err := os.Lstat(path) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err != nil { - return result, fmt.Errorf("inspect cache file %s: %w", path, err) - } - if !os.SameFile(info, currentInfo) || !currentInfo.ModTime().Before(cutoff) { - continue - } if err := os.Remove(path); err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } return result, fmt.Errorf("remove cache file %s: %w", path, err) } } @@ -84,12 +53,3 @@ func PruneCache(deleteFiles bool) (CachePruneResult, error) { return result, nil } - -func isCacheFile(name string, extension string) bool { - if !strings.HasSuffix(name, extension) { - return false - } - - id, err := strconv.Atoi(strings.TrimSuffix(name, extension)) - return err == nil && id > 0 -} From 521416b0305c212bc58318a0a43d2655d9c4b71d Mon Sep 17 00:00:00 2001 From: AiAe Date: Fri, 25 Sep 2026 09:44:49 +0300 Subject: [PATCH 3/4] Remove unused import --- files/cache_prune.go | 1 - 1 file changed, 1 deletion(-) diff --git a/files/cache_prune.go b/files/cache_prune.go index 5663b9c..13a6915 100644 --- a/files/cache_prune.go +++ b/files/cache_prune.go @@ -1,7 +1,6 @@ package files import ( - "errors" "fmt" "os" "path/filepath" From b8e60c0af90bf3429bb04798b1348d61766bbd07 Mon Sep 17 00:00:00 2001 From: AiAe Date: Sat, 26 Sep 2026 17:27:08 +0300 Subject: [PATCH 4/4] Fix webhooks --- webhooks/webhooks.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go index 23921b8..dd615c3 100644 --- a/webhooks/webhooks.go +++ b/webhooks/webhooks.go @@ -381,10 +381,10 @@ func SendClanFirstPlaceWebhook(clan *db.Clan, mapQua *db.MapQua, newScore *db.Cl embed := discord.NewEmbedBuilder(). SetAuthor(fmt.Sprintf("[%v] %v", clan.Tag, clan.Name), - fmt.Sprintf("https://two.quavergame.com/clans/%v", newScore.ClanId), clan.AvatarURL()). + fmt.Sprintf("https://quavergame.com/clans/%v", newScore.ClanId), clan.AvatarURL()). SetDescription("🏆 Achieved a new first place clan score!\n\n"+ - "**Note: Clans is in private beta which is only available to [donators](https://two.quavergame.com/donate).**"). - AddField("Map", fmt.Sprintf("[%v](https://two.quavergame.com/mapset/%v/map/%v?type=clan)", mapQua, mapQua.MapsetId, mapQua.Id), false). + "**Note: Clans is in private beta which is only available to [donators](https://quavergame.com/donate).**"). + AddField("Map", fmt.Sprintf("[%v](https://quavergame.com/mapset/%v/map/%v?type=clans)", mapQua, mapQua.MapsetId, mapQua.Id), false). AddField("Overall Rating", fmt.Sprintf("%.2f", newScore.OverallRating), true). AddField("Overall Accuracy", fmt.Sprintf("%.2f%%", newScore.OverallAccuracy), true). SetImagef("https://cdn.quavergame.com/mapsets/%v.jpg", mapQua.MapsetId). @@ -394,7 +394,7 @@ func SendClanFirstPlaceWebhook(clan *db.Clan, mapQua *db.MapQua, newScore *db.Cl SetColor(0x00FF00) if oldScore != nil { - embed.AddField("Previous #1 Holder", fmt.Sprintf("[[%v] %v](https://two.quavergame.com/clans/%v)", oldScore.Clan.Tag, oldScore.Clan.Name, oldScore.ClanId), false) + embed.AddField("Previous #1 Holder", fmt.Sprintf("[[%v] %v](https://quavergame.com/clans/%v)", oldScore.Clan.Tag, oldScore.Clan.Name, oldScore.ClanId), false) } _, err := clansFirstPlace.CreateMessage(discord.WebhookMessageCreate{ @@ -417,8 +417,8 @@ func SendClanRankedWebhook(mapQua *db.MapQua) error { embed := discord.NewEmbedBuilder(). SetTitle("✅ New Map Clan Ranked!"). SetDescription("A new map has been clan ranked and is now available to get scores on.\n\n"+ - "**Note: Clans is in private beta which is only available to [donators](https://two.quavergame.com/donate).**"). - AddField("Map", fmt.Sprintf("[%v](https://two.quavergame.com/mapset/%v/map/%v?type=clan)", mapQua, mapQua.MapsetId, mapQua.Id), false). + "**Note: Clans is in private beta which is only available to [donators](https://quavergame.com/donate).**"). + AddField("Map", fmt.Sprintf("[%v](https://quavergame.com/mapset/%v/map/%v?type=clans)", mapQua, mapQua.MapsetId, mapQua.Id), false). AddField("Creator", fmt.Sprintf("[%v](https://quavergame.com/user/%v)", mapQua.CreatorUsername, mapQua.CreatorId), true). AddField("Game Mode", enums.GetShorthandGameModeString(mapQua.GameMode), true). AddField("Difficulty ", fmt.Sprintf("%.2f", mapQua.DifficultyRating), true).