Add cache prune cron/command

cmd/console/cmd.go

@@ -38,6 +38,7 @@ func init() {
 
 	// Commands
 	RootCmd.AddCommand(commands.CacheClearCmd)
+	RootCmd.AddCommand(commands.CachePruneCmd)
 	RootCmd.AddCommand(commands.CacheLeaderboardCmd)
 	RootCmd.AddCommand(commands.CacheClanLeaderboard)
 	RootCmd.AddCommand(commands.ElasticIndexMapsets)

cmd/console/commands/cache_prune.go

@@ -0,0 +1,41 @@
+package commands
+
+import (
+	"github.com/Quaver/api2/config"
+	"github.com/Quaver/api2/files"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var CachePruneCmd = &cobra.Command{
+	Use:   "cache:prune",
+	Short: "Finds cached files unused for 90 days",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		deleteFiles, err := cmd.Flags().GetBool("delete")
+		if err != nil {
+			return err
+		}
+		return RunCachePrune(deleteFiles)
+	},
+}
+
+func init() {
+	CachePruneCmd.Flags().Bool("delete", false, "delete old cached files instead of performing a dry run")
+}
+
+func RunCachePrune(deleteFiles bool) error {
+	logrus.Infof("Cache prune scanning %s", config.Instance.Cache.DataDirectory)
+	result, err := files.PruneCache(deleteFiles)
+	megabytes := result.Bytes / 1_000_000
+
+	if deleteFiles {
+		logrus.Infof("Cache prune deleted %d files (%d MB)", result.Files, megabytes)
+	} else {
+		logrus.Infof("Cache prune dry run found %d files (%d MB)", result.Files, megabytes)
+	}
+
+	if err != nil {
+		logrus.Errorf("Error pruning cached files: %v", err)
+	}
+	return err
+}

cmd/cron/main.go

@@ -46,6 +46,7 @@ func main() {
 	registerCronJob(c, jobs.RankClanMap.Job, func() { commands.ClanRankMapCmd.Run(nil, nil) })
 	registerCronJob(c, jobs.DenyOnHoldOneMonth.Job, func() { commands.DenyOnHoldCmd.Run(nil, nil) })
 	registerCronJob(c, jobs.ClanRecalculate.Job, func() { commands.ClanRecalculateCommand.Run(nil, nil) })
+	registerCronJob(c, jobs.CachePrune.Job, func() { commands.RunCachePrune(true) })
 
 	c.Start()
 

config.example.json

@@ -138,6 +138,11 @@
       "enabled": true,
       "name": "Performs a full recalculation on clans",
       "schedule": "0 * * * *"
+    },
+    "cache_prune": {
+      "enabled": true,
+      "name": "Deletes 90+ days older cached maps, mapsets, and replays",
+      "schedule": "0 3 * * *"
     }
   }
-}
+}

config/config.go

@@ -119,6 +119,7 @@ type Config struct {
 		RankClanMap          CronJob `json:"rank_clan_map"`
 		DenyOnHoldOneMonth   CronJob `json:"deny_on_hold_one_month"`
 		ClanRecalculate      CronJob `json:"clan_recalculate"`
+		CachePrune           CronJob `json:"cache_prune"`
 	} `json:"cron"`
 }
 

files/cache.go

@@ -10,8 +10,11 @@ import (
 	"github.com/sirupsen/logrus"
 	"os"
 	"path/filepath"
+	"time"
 )
 
+const cacheAccessUpdateInterval = 24 * time.Hour
+
 // CreateDirectories Creates the directories needed for the cache
 func CreateDirectories() {
 	if err := os.RemoveAll(GetTempDirectory()); err != nil {
@@ -43,14 +46,17 @@ func CacheQuaFile(mapQua *db.MapQua) (string, error) {
 	path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getMapsDirectory(), fileName))
 
 	// Check for existing file & see if md5 hash matches
-	if _, err := os.Stat(path); err == nil {
+	if info, err := os.Stat(path); err == nil {
 		md5, err := GetFileMD5(path)
 
 		if err != nil {
 			return "", err
 		}
 
 		if md5 == mapQua.MD5 {
+			if err := markCacheAccess(path, info); err != nil {
+				logrus.Warnf("Failed to update access time for %v: %v", path, err)
+			}
 			return path, nil
 		}
 	}
@@ -89,14 +95,17 @@ func CacheMapset(mapset *db.Mapset) (string, error) {
 	path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getMapsetDirectory(), fileName))
 
 	// Check MD5 hash of existing file
-	if _, err := os.Stat(path); err == nil {
+	if info, err := os.Stat(path); err == nil {
 		md5, err := GetFileMD5(path)
 
 		if err != nil {
 			return "", err
 		}
 
 		if md5 == mapset.PackageMD5 {
+			if err := markCacheAccess(path, info); err != nil {
+				logrus.Warnf("Failed to update access time for %v: %v", path, err)
+			}
 			return path, nil
 		}
 	}
@@ -113,7 +122,10 @@ func CacheReplay(scoreId int) (string, error) {
 	fileName := fmt.Sprintf("%v.qr", scoreId)
 	path, _ := filepath.Abs(fmt.Sprintf("%v/%v", getReplayDirectory(), fileName))
 
-	if _, err := os.Stat(path); err == nil {
+	if info, err := os.Stat(path); err == nil {
+		if err := markCacheAccess(path, info); err != nil {
+			logrus.Warnf("Failed to update access time for %v: %v", path, err)
+		}
 		return path, nil
 	}
 
@@ -144,3 +156,13 @@ func GetBackupsDirectory() string {
 func GetTempDirectory() string {
 	return "../../temp"
 }
+
+func markCacheAccess(path string, info os.FileInfo) error {
+	now := time.Now()
+
+	if now.Sub(info.ModTime()) < cacheAccessUpdateInterval {
+		return nil
+	}
+
+	return os.Chtimes(path, now, now)
+}

files/cache_prune.go

@@ -0,0 +1,54 @@
+package files
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Quaver/api2/config"
+)
+
+const cacheRetention = 90 * 24 * time.Hour
+
+type CachePruneResult struct {
+	Files int
+	Bytes int64
+}
+
+// PruneCache finds cached maps, mapsets, and replays that are unused for 90 days.
+func PruneCache(deleteFiles bool) (CachePruneResult, error) {
+	var result CachePruneResult
+	root := config.Instance.Cache.DataDirectory
+	cutoff := time.Now().Add(-cacheRetention)
+
+	for _, name := range []string{"maps", "mapsets", "replays"} {
+		directory := filepath.Join(root, name)
+		entries, err := os.ReadDir(directory)
+		if err != nil {
+			return result, fmt.Errorf("read cache directory %s: %w", directory, err)
+		}
+
+		for _, entry := range entries {
+			path := filepath.Join(directory, entry.Name())
+			info, err := entry.Info()
+			if err != nil {
+				return result, fmt.Errorf("inspect cache file %s: %w", path, err)
+			}
+			if !info.Mode().IsRegular() || !info.ModTime().Before(cutoff) {
+				continue
+			}
+
+			if deleteFiles {
+				if err := os.Remove(path); err != nil {
+					return result, fmt.Errorf("remove cache file %s: %w", path, err)
+				}
+			}
+
+			result.Files++
+			result.Bytes += info.Size()
+		}
+	}
+
+	return result, nil
+}

webhooks/webhooks.go

@@ -381,10 +381,10 @@ func SendClanFirstPlaceWebhook(clan *db.Clan, mapQua *db.MapQua, newScore *db.Cl
 
 	embed := discord.NewEmbedBuilder().
 		SetAuthor(fmt.Sprintf("[%v] %v", clan.Tag, clan.Name),
-			fmt.Sprintf("https://two.quavergame.com/clans/%v", newScore.ClanId), clan.AvatarURL()).
+			fmt.Sprintf("https://quavergame.com/clans/%v", newScore.ClanId), clan.AvatarURL()).
 		SetDescription("🏆 Achieved a new first place clan score!\n\n"+
-			"**Note: Clans is in private beta which is only available to [donators](https://two.quavergame.com/donate).**").
-		AddField("Map", fmt.Sprintf("[%v](https://two.quavergame.com/mapset/%v/map/%v?type=clan)", mapQua, mapQua.MapsetId, mapQua.Id), false).
+			"**Note: Clans is in private beta which is only available to [donators](https://quavergame.com/donate).**").
+		AddField("Map", fmt.Sprintf("[%v](https://quavergame.com/mapset/%v/map/%v?type=clans)", mapQua, mapQua.MapsetId, mapQua.Id), false).
 		AddField("Overall Rating", fmt.Sprintf("%.2f", newScore.OverallRating), true).
 		AddField("Overall Accuracy", fmt.Sprintf("%.2f%%", newScore.OverallAccuracy), true).
 		SetImagef("https://cdn.quavergame.com/mapsets/%v.jpg", mapQua.MapsetId).
@@ -394,7 +394,7 @@ func SendClanFirstPlaceWebhook(clan *db.Clan, mapQua *db.MapQua, newScore *db.Cl
 		SetColor(0x00FF00)
 
 	if oldScore != nil {
-		embed.AddField("Previous #1 Holder", fmt.Sprintf("[[%v] %v](https://two.quavergame.com/clans/%v)", oldScore.Clan.Tag, oldScore.Clan.Name, oldScore.ClanId), false)
+		embed.AddField("Previous #1 Holder", fmt.Sprintf("[[%v] %v](https://quavergame.com/clans/%v)", oldScore.Clan.Tag, oldScore.Clan.Name, oldScore.ClanId), false)
 	}
 
 	_, err := clansFirstPlace.CreateMessage(discord.WebhookMessageCreate{
@@ -417,8 +417,8 @@ func SendClanRankedWebhook(mapQua *db.MapQua) error {
 	embed := discord.NewEmbedBuilder().
 		SetTitle("✅ New Map Clan Ranked!").
 		SetDescription("A new map has been clan ranked and is now available to get scores on.\n\n"+
-			"**Note: Clans is in private beta which is only available to [donators](https://two.quavergame.com/donate).**").
-		AddField("Map", fmt.Sprintf("[%v](https://two.quavergame.com/mapset/%v/map/%v?type=clan)", mapQua, mapQua.MapsetId, mapQua.Id), false).
+			"**Note: Clans is in private beta which is only available to [donators](https://quavergame.com/donate).**").
+		AddField("Map", fmt.Sprintf("[%v](https://quavergame.com/mapset/%v/map/%v?type=clans)", mapQua, mapQua.MapsetId, mapQua.Id), false).
 		AddField("Creator", fmt.Sprintf("[%v](https://quavergame.com/user/%v)", mapQua.CreatorUsername, mapQua.CreatorId), true).
 		AddField("Game Mode", enums.GetShorthandGameModeString(mapQua.GameMode), true).
 		AddField("Difficulty ", fmt.Sprintf("%.2f", mapQua.DifficultyRating), true).