Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions internal/requestdb/requestdb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,53 @@ func TestLedgerUsageAndBodies(t *testing.T) {
t.Fatalf("body=%+v err=%v", body, err)
}
}

// TestLedgerTimeZoneChange 验证服务器时区改变后重新打开账本时按新时区重建本地日汇总,小时汇总与原始记录不变
func TestLedgerTimeZoneChange(t *testing.T) {
local := time.Local
t.Cleanup(func() { time.Local = local })
logs := func(level, message string, attrs ...any) {
if level != "INFO" {
t.Errorf("%s %s %v", level, message, attrs)
}
}
path := filepath.Join(t.TempDir(), "requests.db")
time.Local = time.FixedZone("UTC+8", 8*60*60)
store, err := Open(path, logs)
if err != nil {
t.Fatal(err)
}
midnight := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour)
// at 在 UTC+8 是 midnight 当日 23:30,在 UTC+9 是次日 00:30
at := midnight.Add(15*time.Hour + 30*time.Minute)
store.Record(Row{ID: "a", Time: at, Model: "flash", State: "completed", Duration: 2 * time.Second, InputTokens: 10, TotalTokens: 30})
if err := store.Close(); err != nil {
t.Fatal(err)
}
zone := time.FixedZone("UTC+9", 9*60*60)
time.Local = zone
store, err = Open(path, logs)
if err != nil {
t.Fatal(err)
}
defer store.Close()
usage := func(from, to time.Time, bucket time.Duration) api.UsageReport {
t.Helper()
report, err := store.Usage(context.Background(), api.UsageQuery{From: from, To: to, Bucket: bucket, Location: zone, Stack: "model"})
if err != nil {
t.Fatal(err)
}
return report
}
// day 是 at 在 UTC+9 所在本地日的起点
day := midnight.Add(15 * time.Hour)
if report := usage(day, day.Add(24*time.Hour), 24*time.Hour); report.Totals.Requests != 1 || report.Totals.TotalTokens != 30 || report.Previous.Requests != 0 {
t.Fatalf("本地日 totals=%+v previous=%+v", report.Totals, report.Previous)
}
if report := usage(day, day.Add(time.Hour), time.Hour); report.Totals.Requests != 1 || report.Totals.Duration.AvgMS != 2000 {
t.Fatalf("小时 totals=%+v", report.Totals)
}
if report := usage(at.Add(-time.Minute), at.Add(time.Minute), time.Minute); report.Totals.Requests != 1 || report.Totals.InputTokens != 10 {
t.Fatalf("原始记录 totals=%+v", report.Totals)
}
}
43 changes: 30 additions & 13 deletions internal/sqlitedb/sqlitedb.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,11 @@ import (
"github.com/mattn/go-sqlite3"
)

// driverName 是每个连接都设置好收缩、日志与等待参数的 SQLite 驱动名
// driverName 是每个连接都设置好日志与等待参数的 SQLite 驱动名
const driverName = "sqlite3_aistudio2api"

// connectPragmas 是每个新连接执行的设置,增量收缩须在启用 WAL 之前设置
const connectPragmas = `PRAGMA auto_vacuum = INCREMENTAL;
PRAGMA busy_timeout = 10000;
// connectPragmas 是每个新连接执行的设置
const connectPragmas = `PRAGMA busy_timeout = 10000;
PRAGMA journal_mode = WAL;
PRAGMA synchronous = NORMAL;
PRAGMA journal_size_limit = 524288;`
Expand All @@ -32,20 +31,14 @@ func init() {
}})
}

// Open 打开或创建数据库,为已有表补上缺少的列、执行 schema 并删除 schema 没有定义的对象,非增量收缩的已有库先整理为增量收缩,最后交还空闲页
// Open 打开或创建数据库,新库与非增量收缩的已有库先整理为增量收缩,再为已有表补上缺少的列、执行 schema 并删除 schema 没有定义的对象,最后交还空闲页
func Open(path string, schema string) (*sql.DB, error) {
db, err := sql.Open(driverName, "file:"+filepath.ToSlash(path))
if err != nil {
return nil, err
}
var mode int
if err := db.QueryRow(`PRAGMA auto_vacuum`).Scan(&mode); err != nil {
return nil, errors.Join(fmt.Errorf("读取收缩模式: %w", err), db.Close())
}
if mode != incrementalVacuum {
if _, err := db.Exec(`VACUUM`); err != nil {
return nil, errors.Join(fmt.Errorf("整理数据库: %w", err), db.Close())
}
if err := useIncrementalVacuum(db); err != nil {
return nil, errors.Join(err, db.Close())
}
if err := applySchema(db, schema); err != nil {
return nil, errors.Join(fmt.Errorf("同步表结构: %w", err), db.Close())
Expand All @@ -56,6 +49,30 @@ func Open(path string, schema string) (*sql.DB, error) {
return db, nil
}

// useIncrementalVacuum 在同一连接上把不是增量收缩的库设为增量收缩并整理一次
func useIncrementalVacuum(db *sql.DB) error {
ctx := context.Background()
conn, err := db.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
var mode int
if err := conn.QueryRowContext(ctx, `PRAGMA auto_vacuum`).Scan(&mode); err != nil {
return fmt.Errorf("读取收缩模式: %w", err)
}
if mode == incrementalVacuum {
return nil
}
if _, err := conn.ExecContext(ctx, `PRAGMA auto_vacuum = INCREMENTAL`); err != nil {
return fmt.Errorf("设置收缩模式: %w", err)
}
if _, err := conn.ExecContext(ctx, `VACUUM`); err != nil {
return fmt.Errorf("整理数据库: %w", err)
}
return nil
}

// applySchema 为已有表补上 schema 定义而缺少的列,执行 schema,再删除 schema 没有定义的表、索引、视图与触发器
func applySchema(db *sql.DB, schema string) error {
defined, err := sql.Open(driverName, ":memory:")
Expand Down