diff --git a/cmd/fsb/run.go b/cmd/fsb/run.go index 913371a3e..be1f22c77 100644 --- a/cmd/fsb/run.go +++ b/cmd/fsb/run.go @@ -32,6 +32,7 @@ func runApp(cmd *cobra.Command, args []string) { config.Load(utils.Logger, cmd) // reinitialize with correct dev mode utils.InitLogger(config.ValueOf.Dev) + utils.InitSendPacer(time.Duration(config.ValueOf.SendDelayMs) * time.Millisecond) log := utils.Logger mainLogger := log.Named("Main") mainLogger.Info("Starting server") diff --git a/config/config.go b/config/config.go index 62afe24b7..c55aac86c 100644 --- a/config/config.go +++ b/config/config.go @@ -57,6 +57,13 @@ type config struct { StreamBufferCount int `envconfig:"STREAM_BUFFER_COUNT" default:"8"` StreamTimeoutSec int `envconfig:"STREAM_TIMEOUT_SEC" default:"30"` StreamMaxRetries int `envconfig:"STREAM_MAX_RETRIES" default:"3"` + + // telegram flood control + FloodWaitMaxRetries int `envconfig:"FLOOD_WAIT_MAX_RETRIES" default:"10"` + FloodWaitMaxWaitSec int `envconfig:"FLOOD_WAIT_MAX_WAIT_SEC" default:"300"` + RateLimitIntervalMs int `envconfig:"RATE_LIMIT_INTERVAL_MS" default:"100"` + RateLimitBurst int `envconfig:"RATE_LIMIT_BURST" default:"5"` + SendDelayMs int `envconfig:"SEND_DELAY_MS" default:"250"` } var botTokenRegex = regexp.MustCompile(`MULTI\_TOKEN\d+=(.*)`) @@ -94,6 +101,11 @@ func SetFlagsFromConfig(cmd *cobra.Command) { cmd.Flags().Int("stream-buffer-count", ValueOf.StreamBufferCount, "Number of blocks to prefetch") cmd.Flags().Int("stream-timeout-sec", ValueOf.StreamTimeoutSec, "Maximum time to wait for a single block (in seconds)") cmd.Flags().Int("stream-max-retries", ValueOf.StreamMaxRetries, "Number of retry attempts for failed fetches") + cmd.Flags().Int("flood-wait-max-retries", ValueOf.FloodWaitMaxRetries, "Maximum retries on Telegram FLOOD_WAIT errors (0 means retry forever)") + cmd.Flags().Int("flood-wait-max-wait-sec", ValueOf.FloodWaitMaxWaitSec, "Maximum single FLOOD_WAIT duration to sleep for, in seconds") + cmd.Flags().Int("rate-limit-interval-ms", ValueOf.RateLimitIntervalMs, "Minimum interval between Telegram RPCs, in milliseconds") + cmd.Flags().Int("rate-limit-burst", ValueOf.RateLimitBurst, "Burst size of the Telegram RPC rate limiter") + cmd.Flags().Int("send-delay-ms", ValueOf.SendDelayMs, "Minimum delay between link replies for the same chat, in milliseconds") } func (c *config) loadConfigFromArgs(log *zap.Logger, cmd *cobra.Command) { @@ -162,6 +174,21 @@ func (c *config) loadConfigFromArgs(log *zap.Logger, cmd *cobra.Command) { if streamMaxRetries != 0 { os.Setenv("STREAM_MAX_RETRIES", strconv.Itoa(streamMaxRetries)) } + for _, name := range []string{ + "flood-wait-max-retries", + "flood-wait-max-wait-sec", + "rate-limit-interval-ms", + "rate-limit-burst", + "send-delay-ms", + } { + // Explicitly set flags win over the env file, including the value 0, + // which is a valid setting for the flood control knobs. + if !cmd.Flags().Changed(name) { + continue + } + value, _ := cmd.Flags().GetInt(name) + os.Setenv(strings.ToUpper(strings.ReplaceAll(name, "-", "_")), strconv.Itoa(value)) + } } func (c *config) setupEnvVars(log *zap.Logger, cmd *cobra.Command) { @@ -231,6 +258,26 @@ func Load(log *zap.Logger, cmd *cobra.Command) { log.Sugar().Info("STREAM_MAX_RETRIES must be greater than 0, defaulting to 3") ValueOf.StreamMaxRetries = 3 } + if ValueOf.FloodWaitMaxRetries < 0 { + log.Sugar().Info("FLOOD_WAIT_MAX_RETRIES can't be negative, retrying forever") + ValueOf.FloodWaitMaxRetries = 0 + } + if ValueOf.FloodWaitMaxWaitSec <= 0 { + log.Sugar().Info("FLOOD_WAIT_MAX_WAIT_SEC must be greater than 0, defaulting to 300 seconds") + ValueOf.FloodWaitMaxWaitSec = 300 + } + if ValueOf.RateLimitIntervalMs <= 0 { + log.Sugar().Info("RATE_LIMIT_INTERVAL_MS must be greater than 0, defaulting to 100") + ValueOf.RateLimitIntervalMs = 100 + } + if ValueOf.RateLimitBurst <= 0 { + log.Sugar().Info("RATE_LIMIT_BURST must be greater than 0, defaulting to 5") + ValueOf.RateLimitBurst = 5 + } + if ValueOf.SendDelayMs < 0 { + log.Sugar().Info("SEND_DELAY_MS can't be negative, defaulting to 250") + ValueOf.SendDelayMs = 250 + } } func getIP(public bool) (string, error) { diff --git a/fsb.sample.env b/fsb.sample.env index 33f1dcea5..46377263e 100644 --- a/fsb.sample.env +++ b/fsb.sample.env @@ -51,6 +51,21 @@ STREAM_BUFFER_COUNT=8 STREAM_TIMEOUT_SEC=30 STREAM_MAX_RETRIES=3 +# Telegram flood control +# Telegram answers bursts (e.g. a media group of 8 photos = 8 forwards + 8 replies) +# with 420 FLOOD_WAIT. These knobs decide how the bot reacts. +# +# FLOOD_WAIT_MAX_RETRIES = retries per request after a FLOOD_WAIT (0 = retry forever) +# FLOOD_WAIT_MAX_WAIT_SEC = longest single FLOOD_WAIT the bot is willing to sleep through +# RATE_LIMIT_INTERVAL_MS = minimum spacing between two Telegram RPCs +# RATE_LIMIT_BURST = how many RPCs may go out back to back +# SEND_DELAY_MS = minimum delay between two link replies in the same chat (0 = off) +FLOOD_WAIT_MAX_RETRIES=10 +FLOOD_WAIT_MAX_WAIT_SEC=300 +RATE_LIMIT_INTERVAL_MS=100 +RATE_LIMIT_BURST=5 +SEND_DELAY_MS=250 + # For muti token support # Refer https://github.com/EverythingSuckz/TG-FileStreamBot/tree/golang#use-multiple-bots-to-speed-up diff --git a/internal/bot/client.go b/internal/bot/client.go index df9450b13..50d9b01b8 100644 --- a/internal/bot/client.go +++ b/internal/bot/client.go @@ -32,6 +32,7 @@ func StartClient(log *zap.Logger) (*gotgproto.Client, error) { sqlite.Open("fsb.session"), ), DisableCopyright: true, + Middlewares: GetFloodMiddleware(log), }, ) resultChan <- struct { diff --git a/internal/bot/middleware.go b/internal/bot/middleware.go index 7d9dd400f..3b0642334 100644 --- a/internal/bot/middleware.go +++ b/internal/bot/middleware.go @@ -1,20 +1,84 @@ package bot import ( + "context" + "fmt" + "strings" "time" + "EverythingSuckz/fsb/config" + "github.com/gotd/contrib/middleware/floodwait" "github.com/gotd/contrib/middleware/ratelimit" + "github.com/gotd/td/bin" "github.com/gotd/td/telegram" + "github.com/gotd/td/tg" + "github.com/gotd/td/tgerr" "go.uber.org/zap" "golang.org/x/time/rate" ) +// floodWaitLogger reports FLOOD_WAIT answers from Telegram. It sits between the +// waiter and the rate limiter, so every single attempt that got throttled is +// logged, not just the final failure. +type floodWaitLogger struct { + log *zap.Logger +} + +func (m floodWaitLogger) Handle(next tg.Invoker) telegram.InvokeFunc { + return func(ctx context.Context, input bin.Encoder, output bin.Decoder) error { + err := next.Invoke(ctx, input, output) + if d, ok := tgerr.AsFloodWait(err); ok { + m.log.Warn("Telegram asked to wait, retrying", + zap.String("method", methodName(input)), + zap.Duration("wait", d), + ) + } + return err + } +} + +// methodName strips the generated type decoration, e.g. +// *tg.MessagesForwardMessagesRequest becomes MessagesForwardMessages. +func methodName(input bin.Encoder) string { + name := fmt.Sprintf("%T", input) + name = strings.TrimPrefix(name, "*tg.") + name = strings.TrimSuffix(name, "Request") + return name +} + +// GetFloodMiddleware returns the middleware chain used by every Telegram +// client (main bot, workers and userbot). +// +// Order matters: middlewares are applied in slice order, so the waiter is the +// outermost one and its retries go through the logger and the rate limiter +// again. That way a retried request is both logged and rate limited. func GetFloodMiddleware(log *zap.Logger) []telegram.Middleware { - waiter := floodwait.NewSimpleWaiter().WithMaxRetries(10) - ratelimiter := ratelimit.New(rate.Every(time.Millisecond*100), 5) + return newFloodMiddleware( + log.Named("FloodWait"), + uint(config.ValueOf.FloodWaitMaxRetries), + time.Duration(config.ValueOf.FloodWaitMaxWaitSec)*time.Second, + time.Duration(config.ValueOf.RateLimitIntervalMs)*time.Millisecond, + config.ValueOf.RateLimitBurst, + ) +} + +// newFloodMiddleware builds the chain from explicit values so it can be +// exercised in tests without touching the global config. +func newFloodMiddleware( + log *zap.Logger, + maxRetries uint, + maxWait time.Duration, + interval time.Duration, + burst int, +) []telegram.Middleware { + waiter := floodwait.NewSimpleWaiter(). + WithMaxRetries(maxRetries). + WithMaxWait(maxWait) + limiter := ratelimit.New(rate.Every(interval), burst) return []telegram.Middleware{ waiter, - ratelimiter, + floodWaitLogger{log: log}, + limiter, } } diff --git a/internal/bot/middleware_test.go b/internal/bot/middleware_test.go new file mode 100644 index 000000000..3cc6bb85d --- /dev/null +++ b/internal/bot/middleware_test.go @@ -0,0 +1,65 @@ +package bot + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/gotd/td/bin" + "github.com/gotd/td/telegram" + "github.com/gotd/td/tg" + "github.com/gotd/td/tgerr" + "go.uber.org/zap" +) + +func chain(middlewares []telegram.Middleware, next tg.Invoker) tg.Invoker { + for i := len(middlewares) - 1; i >= 0; i-- { + next = middlewares[i].Handle(next) + } + return next +} + +func TestFloodMiddlewareRetriesFloodWait(t *testing.T) { + var attempts atomic.Int32 + invoker := telegram.InvokeFunc(func(ctx context.Context, input bin.Encoder, output bin.Decoder) error { + if attempts.Add(1) < 2 { + return tgerr.New(420, "FLOOD_WAIT_0") + } + return nil + }) + + mw := newFloodMiddleware(zap.NewNop(), 5, time.Minute, time.Millisecond, 1) + if err := chain(mw, invoker).Invoke(context.Background(), &tg.HelpGetConfigRequest{}, nil); err != nil { + t.Fatalf("invoke returned error: %v", err) + } + if got := attempts.Load(); got != 2 { + t.Fatalf("attempts = %d, want 2", got) + } +} + +func TestFloodMiddlewareHonoursMaxWait(t *testing.T) { + var attempts atomic.Int32 + invoker := telegram.InvokeFunc(func(ctx context.Context, input bin.Encoder, output bin.Decoder) error { + attempts.Add(1) + return tgerr.New(420, "FLOOD_WAIT_600") + }) + + mw := newFloodMiddleware(zap.NewNop(), 5, 10*time.Second, time.Millisecond, 1) + err := chain(mw, invoker).Invoke(context.Background(), &tg.HelpGetConfigRequest{}, nil) + if err == nil { + t.Fatal("invoke returned nil, want error") + } + if _, ok := tgerr.AsFloodWait(err); !ok { + t.Fatalf("error %v does not wrap the flood wait", err) + } + if got := attempts.Load(); got != 1 { + t.Fatalf("attempts = %d, want 1", got) + } +} + +func TestMethodName(t *testing.T) { + if got := methodName(&tg.MessagesForwardMessagesRequest{}); got != "MessagesForwardMessages" { + t.Fatalf("methodName = %q", got) + } +} diff --git a/internal/bot/userbot.go b/internal/bot/userbot.go index 2813709fa..dd682d26c 100644 --- a/internal/bot/userbot.go +++ b/internal/bot/userbot.go @@ -31,6 +31,7 @@ func StartUserBot(l *zap.Logger) { &gotgproto.ClientOpts{ Session: sessionMaker.PyrogramSession(config.ValueOf.UserSession), DisableCopyright: true, + Middlewares: GetFloodMiddleware(log), }, ) if err != nil { diff --git a/internal/commands/stream.go b/internal/commands/stream.go index 0498f2576..83c01a1f8 100644 --- a/internal/commands/stream.go +++ b/internal/commands/stream.go @@ -58,10 +58,20 @@ func sendLink(ctx *ext.Context, u *ext.Update) error { ctx.Reply(u, ext.ReplyTextString("Sorry, this message type is unsupported."), nil) return dispatcher.EndGroups } + // A media group arrives as one update per message, so keep a minimum + // distance between two answers in the same chat to avoid FLOOD_WAIT. + // One Wait per message: the forward is paced, and Done moves the anchor to + // the moment the reply is out, so the delay lands between two replies + // instead of between their starts. + if err := utils.SendPacer.Wait(ctx.Context, chatId); err != nil { + utils.Logger.Sugar().Debug(err) + return dispatcher.EndGroups + } + defer utils.SendPacer.Done(chatId) update, err := utils.ForwardMessages(ctx, chatId, config.ValueOf.LogChannelID, u.EffectiveMessage.ID) if err != nil { utils.Logger.Sugar().Error(err) - ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("Error - %s", err.Error())), nil) + ctx.Reply(u, ext.ReplyTextString(errorText(err)), nil) return dispatcher.EndGroups } if len(update.Updates) < 2 { @@ -130,7 +140,16 @@ func sendLink(ctx *ext.Context, u *ext.Update) error { } if err != nil { utils.Logger.Sugar().Error(err) - ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("Error - %s", err.Error())), nil) + ctx.Reply(u, ext.ReplyTextString(errorText(err)), nil) } return dispatcher.EndGroups } + +// errorText renders err for the user. Flood waits get an actionable message +// instead of the raw RPC error, everything else is passed through. +func errorText(err error) string { + if hint, ok := utils.DescribeFloodWait(err); ok { + return hint + } + return fmt.Sprintf("Error - %s", err.Error()) +} diff --git a/internal/utils/errors.go b/internal/utils/errors.go new file mode 100644 index 000000000..068bc795b --- /dev/null +++ b/internal/utils/errors.go @@ -0,0 +1,27 @@ +package utils + +import ( + "fmt" + "time" + + "github.com/gotd/td/tgerr" +) + +// DescribeFloodWait turns a FLOOD_WAIT error into a message a user can act on. +// It returns false when err is not a flood wait. +func DescribeFloodWait(err error) (string, bool) { + if err == nil { + return "", false + } + wait, ok := tgerr.AsFloodWait(err) + if !ok { + return "", false + } + if wait <= 0 { + wait = time.Second + } + return fmt.Sprintf( + "Telegram is rate limiting this bot, please try again in %s.", + wait.Round(time.Second), + ), true +} diff --git a/internal/utils/errors_test.go b/internal/utils/errors_test.go new file mode 100644 index 000000000..2070f28be --- /dev/null +++ b/internal/utils/errors_test.go @@ -0,0 +1,25 @@ +package utils + +import ( + "errors" + "strings" + "testing" + + "github.com/gotd/td/tgerr" +) + +func TestDescribeFloodWait(t *testing.T) { + if _, ok := DescribeFloodWait(nil); ok { + t.Fatal("nil error described as flood wait") + } + if _, ok := DescribeFloodWait(errors.New("boom")); ok { + t.Fatal("generic error described as flood wait") + } + hint, ok := DescribeFloodWait(tgerr.New(420, "FLOOD_WAIT_7")) + if !ok { + t.Fatal("flood wait not detected") + } + if !strings.Contains(hint, "7s") { + t.Fatalf("hint = %q, want the wait duration", hint) + } +} diff --git a/internal/utils/pacing.go b/internal/utils/pacing.go new file mode 100644 index 000000000..b67dc543c --- /dev/null +++ b/internal/utils/pacing.go @@ -0,0 +1,95 @@ +package utils + +import ( + "context" + "sync" + "time" +) + +// ChatPacer keeps at least a fixed delay between operations that belong to the +// same chat. +// +// A media group is delivered as one update per message, so a single album of 8 +// photos makes the bot forward 8 messages and send 8 replies in a couple of +// seconds. Telegram answers that burst with 420 FLOOD_WAIT. The retry +// middleware absorbs those answers, this pacer keeps most of them from +// happening in the first place. +type ChatPacer struct { + interval time.Duration + + mu sync.Mutex + next map[int64]time.Time +} + +// NewChatPacer returns a pacer that allows one operation per interval for a +// given chat. A non-positive interval disables pacing. +func NewChatPacer(interval time.Duration) *ChatPacer { + return &ChatPacer{ + interval: interval, + next: make(map[int64]time.Time), + } +} + +// Wait blocks until the next operation for chatID is allowed to run and +// reserves its slot. +func (p *ChatPacer) Wait(ctx context.Context, chatID int64) error { + if p == nil || p.interval <= 0 { + return nil + } + now := time.Now() + + p.mu.Lock() + if len(p.next) > 4096 { + for id, at := range p.next { + if at.Before(now) { + delete(p.next, id) + } + } + } + at := p.next[chatID] + if at.Before(now) { + at = now + } + p.next[chatID] = at.Add(p.interval) + p.mu.Unlock() + + delay := time.Until(at) + if delay <= 0 { + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Done records that the operation started by Wait for chatID has finished. +// +// Wait reserves the slot when the operation starts, so a slow forward or a +// flood wait retry would eat the whole interval and the next message would +// start immediately after it. Done moves the anchor to the moment the reply +// was sent, which makes interval the real gap between two finished replies. +// Callers use one Wait and one Done per message, never two Waits. +func (p *ChatPacer) Done(chatID int64) { + if p == nil || p.interval <= 0 { + return + } + at := time.Now().Add(p.interval) + p.mu.Lock() + defer p.mu.Unlock() + if at.After(p.next[chatID]) { + p.next[chatID] = at + } +} + +// SendPacer is the process wide pacer used before answering a user. +var SendPacer = NewChatPacer(0) + +// InitSendPacer configures SendPacer from the loaded configuration. +func InitSendPacer(interval time.Duration) { + SendPacer = NewChatPacer(interval) +} diff --git a/internal/utils/pacing_test.go b/internal/utils/pacing_test.go new file mode 100644 index 000000000..52bc9dd0a --- /dev/null +++ b/internal/utils/pacing_test.go @@ -0,0 +1,124 @@ +package utils + +import ( + "context" + "sync" + "testing" + "time" +) + +func TestChatPacerConcurrentSameChat(t *testing.T) { + p := NewChatPacer(10 * time.Millisecond) + start := time.Now() + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + if err := p.Wait(context.Background(), 1); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if d := time.Since(start); d < 70*time.Millisecond { + t.Fatalf("8 concurrent waits took %v, want at least 70ms", d) + } +} + +func TestChatPacerSpacesSameChat(t *testing.T) { + p := NewChatPacer(20 * time.Millisecond) + start := time.Now() + for i := range 4 { + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("wait %d: %v", i, err) + } + } + if d := time.Since(start); d < 60*time.Millisecond { + t.Fatalf("4 waits took %v, want at least 60ms", d) + } +} + +func TestChatPacerSeparatesChats(t *testing.T) { + p := NewChatPacer(time.Hour) + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("first wait: %v", err) + } + start := time.Now() + if err := p.Wait(context.Background(), 2); err != nil { + t.Fatalf("second wait: %v", err) + } + if d := time.Since(start); d > time.Second { + t.Fatalf("other chat was delayed by %v", d) + } +} + +func TestChatPacerStopsOnCanceledContext(t *testing.T) { + p := NewChatPacer(time.Hour) + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("first wait: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.Wait(ctx, 1); err == nil { + t.Fatal("wait returned nil on canceled context") + } +} + +func TestChatPacerDisabled(t *testing.T) { + p := NewChatPacer(0) + start := time.Now() + for i := range 100 { + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("wait %d: %v", i, err) + } + } + if d := time.Since(start); d > time.Second { + t.Fatalf("disabled pacer blocked for %v", d) + } +} + +// Done must move the anchor to the end of the work, so the delay is the gap +// between two finished replies and not between their starts. +func TestChatPacerDoneAnchorsToCompletion(t *testing.T) { + const interval = 50 * time.Millisecond + p := NewChatPacer(interval) + + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("first wait: %v", err) + } + time.Sleep(30 * time.Millisecond) // simulate forward + reply + p.Done(1) + + start := time.Now() + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("second wait: %v", err) + } + d := time.Since(start) + if d < 45*time.Millisecond { + t.Fatalf("second wait took %v, want the full interval after completion", d) + } + // One Wait plus one Done must not stack two intervals. + if d > 2*interval { + t.Fatalf("second wait took %v, want at most one interval", d) + } +} + +func TestChatPacerDoneWithoutWait(t *testing.T) { + p := NewChatPacer(20 * time.Millisecond) + p.Done(1) // no reservation, must not panic + start := time.Now() + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("wait: %v", err) + } + if d := time.Since(start); d < 15*time.Millisecond { + t.Fatalf("wait after Done took %v, want the interval", d) + } + p.Done(1) +} + +func TestChatPacerDoneDisabled(t *testing.T) { + p := NewChatPacer(0) + p.Done(1) + if err := p.Wait(context.Background(), 1); err != nil { + t.Fatalf("wait: %v", err) + } +}