From ce57a1603b6623a43b9dd8af9a6e2d2c88f2760d Mon Sep 17 00:00:00 2001 From: crispinhu Date: Thu, 19 Nov 2026 22:24:00 +0800 Subject: [PATCH 1/3] fix(runner): install the Linux Docker helper on macOS --- deploy/bind-cli.sh | 63 +++++++++++--- deploy/bind_cli_docker_runner_test.go | 117 ++++++++++++++++++++++++++ docs/deployment.md | 2 + internal/runner/daemon.go | 2 + internal/runner/update.go | 52 +++++++++--- internal/runner/update_test.go | 64 ++++++++++++++ internal/runtime/mcp_call.go | 21 +++-- 7 files changed, 290 insertions(+), 31 deletions(-) create mode 100644 deploy/bind_cli_docker_runner_test.go diff --git a/deploy/bind-cli.sh b/deploy/bind-cli.sh index a9d0233c2..d3bf6b696 100755 --- a/deploy/bind-cli.sh +++ b/deploy/bind-cli.sh @@ -25,7 +25,7 @@ # --uninstall 卸载:通知平台注销机器、停止服务、删除全部文件和配置 # # 默认直接从平台下载对应 OS/架构的二进制(由服务端 /runners/agentflow-runner 分发)。 -# 脚本安装 agentflow-runner 和跨沙箱 Gate MCP helper、写配置并启动 +# 脚本安装 agentflow-runner 和跨沙箱 Git/MCP、Gate MCP helper、写配置并启动 # daemon。Runner 仅向平台发起出站 HTTPS,不开放任何入站端口。 set -euo pipefail @@ -343,6 +343,7 @@ RUNNER_HOME="${HOME}/.agentflow-runner" ENV_FILE="${RUNNER_HOME}/env" STATE_FILE="${RUNNER_HOME}/state.json" BINARY="${BIN_DIR}/agentflow-runner" +DOCKER_RUNNER_BINARY="${BIN_DIR}/agentflow-runner-linux-${ARCH}" GATE_BINARY_PATTERN="${BIN_DIR}/agentflow-gate-mcp-{os}-{arch}" PLIST="${HOME}/Library/LaunchAgents/com.agentflow.runner.plist" UNIT="${HOME}/.config/systemd/user/agentflow-runner.service" @@ -421,6 +422,9 @@ if [[ "$UNINSTALL" == "1" ]]; then info "已清理 runner Docker 沙箱和临时镜像" fi rm -f "$BINARY" + if [[ "$OS" == "darwin" ]]; then + rm -f "$DOCKER_RUNNER_BINARY" + fi rm -f "${BIN_DIR}"/agentflow-gate-mcp-darwin-amd64 \ "${BIN_DIR}"/agentflow-gate-mcp-darwin-arm64 \ "${BIN_DIR}"/agentflow-gate-mcp-linux-amd64 \ @@ -537,6 +541,31 @@ case "$(printf '%s' "${AGENTFLOW_BROWSER:-}" | tr '[:upper:]' '[:lower:]')" in esac # ---------- 1. 安装二进制 ---------- +download_runner_binary() { + local target_os="$1" url + if [[ -n "$DOWNLOAD_URL" && ( "$target_os" == "$OS" || "$DOWNLOAD_URL" == *"{os}"* ) ]]; then + url="${DOWNLOAD_URL//\{os\}/$target_os}" + url="${url//\{arch\}/$ARCH}" + else + url="${SERVER}/runners/agentflow-runner?os=${target_os}&arch=${ARCH}" + fi + info "从 ${url} 下载 agentflow-runner(${target_os}/${ARCH})..." + curl -fsSL "$url" -o "$TEMP_BINARY" && [[ -s "$TEMP_BINARY" ]] +} + +build_runner_binary() { + local target_os="$1" source="$2" + info "从源码 ${source} 构建 agentflow-runner(${target_os}/${ARCH})..." + (cd "$source" && CGO_ENABLED=0 GOOS="$target_os" GOARCH="$ARCH" go build -o "$TEMP_BINARY" ./cmd/agentflow-runner) +} + +publish_runner_binary() { + [[ -s "$TEMP_BINARY" ]] || fail "下载或构建的 runner 二进制为空" + chmod +x "$TEMP_BINARY" + mv -f "$TEMP_BINARY" "$1" + TEMP_BINARY="" +} + install_binary() { mkdir -p "$BIN_DIR" TEMP_BINARY=$(mktemp "${BIN_DIR}/.agentflow-runner.XXXXXX") @@ -544,14 +573,10 @@ install_binary() { [[ -f "$LOCAL_BINARY" ]] || fail "本地二进制不存在: $LOCAL_BINARY" cp "$LOCAL_BINARY" "$TEMP_BINARY" elif [[ -n "$DOWNLOAD_URL" ]]; then - URL="${DOWNLOAD_URL//\{os\}/$OS}" - URL="${URL//\{arch\}/$ARCH}" - info "从 ${URL} 下载二进制..." - curl -fsSL "$URL" -o "$TEMP_BINARY" + download_runner_binary "$OS" else # 默认:平台直接分发对应 OS/架构的二进制,无需本机 Go 工具链。 - info "从 ${SERVER} 下载 agentflow-runner(${OS}/${ARCH})..." - if curl -fsSL "${SERVER}/runners/agentflow-runner?os=${OS}&arch=${ARCH}" -o "$TEMP_BINARY"; then + if download_runner_binary "$OS"; then : else # 平台未配置分发:回退到本地源码构建(--source 或当前目录为源码 checkout)。 @@ -564,15 +589,25 @@ install_binary() { 3) 本命令追加 --from <已编译的 agentflow-runner 二进制路径>" fi command -v go >/dev/null 2>&1 || fail "本机未安装 Go,无法从源码构建;请使用 --from 提供二进制" - info "从源码 ${SRC} 构建(${OS}/${ARCH})..." - (cd "$SRC" && CGO_ENABLED=0 GOOS="$OS" GOARCH="$ARCH" go build -o "$TEMP_BINARY" ./cmd/agentflow-runner) \ + build_runner_binary "$OS" "$SRC" \ || fail "源码构建失败;请检查 Go 版本(需 1.22+)或改用 --from" fi fi - [[ -s "$TEMP_BINARY" ]] || fail "下载或构建的 runner 二进制为空" - chmod +x "$TEMP_BINARY" - mv -f "$TEMP_BINARY" "$BINARY" + publish_runner_binary "$BINARY" +} + +install_docker_runner_binary() { + local source="${SOURCE_DIR:-$PWD}" + TEMP_BINARY=$(mktemp "${BIN_DIR}/.agentflow-runner.XXXXXX") + if download_runner_binary linux || + { [[ -d "$source/cmd/agentflow-runner" ]] && command -v go >/dev/null 2>&1 && + build_runner_binary linux "$source" && [[ -s "$TEMP_BINARY" ]]; }; then + publish_runner_binary "$DOCKER_RUNNER_BINARY" + return + fi + rm -f "$TEMP_BINARY" TEMP_BINARY="" + warn "无法获取 Linux/${ARCH} Git/MCP helper,Docker 工作区需要该文件:${DOCKER_RUNNER_BINARY}" } TEMP_BINARY="" @@ -597,11 +632,11 @@ install_gate_binary() { chmod +x "$target" } -# Host sandbox uses the native helper. Docker always uses Linux; on a macOS -# runner both artifacts are therefore required. +# Docker on macOS needs Linux versions of both the Gate MCP and Git/MCP helpers. install_gate_binary "$OS" "$ARCH" if [[ "$OS" != "linux" ]]; then install_gate_binary linux "$ARCH" + install_docker_runner_binary fi if [[ "$UPGRADE" == "1" ]]; then diff --git a/deploy/bind_cli_docker_runner_test.go b/deploy/bind_cli_docker_runner_test.go new file mode 100644 index 000000000..9c3899a1b --- /dev/null +++ b/deploy/bind_cli_docker_runner_test.go @@ -0,0 +1,117 @@ +package deploy + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { + curl, err := exec.LookPath("curl") + if err != nil { + t.Skip("curl is required for the runner installer") + } + for _, tc := range []struct { + name, hostOS, arch, unameArch string + upgrade, missingArtifact bool + }{ + {name: "macOS_install", hostOS: "darwin", arch: "arm64", unameArch: "arm64"}, + {name: "macOS_upgrade", hostOS: "darwin", arch: "amd64", unameArch: "x86_64", upgrade: true}, + {name: "Linux_install", hostOS: "linux", arch: "arm64", unameArch: "aarch64"}, + {name: "macOS_preserves_existing_helper_on_failed_download", hostOS: "darwin", arch: "arm64", unameArch: "arm64", missingArtifact: true}, + } { + t.Run(tc.name, func(t *testing.T) { + fixtureHome := t.TempDir() + bin := filepath.Join(fixtureHome, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + uname := fmt.Sprintf("#!/bin/sh\ncase \"$1\" in -s) echo %s ;; -m) echo %s ;; esac\n", tc.hostOS, tc.unameArch) + if err := os.WriteFile(filepath.Join(bin, "uname"), []byte(uname), 0o755); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + targetOS, arch := req.URL.Query().Get("os"), req.URL.Query().Get("arch") + if arch != tc.arch || (targetOS != tc.hostOS && targetOS != "linux") { + http.NotFound(w, req) + return + } + switch req.URL.Path { + case "/runners/agentflow-runner": + if tc.missingArtifact && targetOS == "linux" { + http.NotFound(w, req) + return + } + fmt.Fprintf(w, "runner %s/%s", targetOS, arch) + case "/runners/agentflow-gate-mcp": + fmt.Fprintf(w, "gate %s/%s", targetOS, arch) + default: + http.NotFound(w, req) + } + })) + defer server.Close() + args := []string{"bind-cli.sh", "--bin-dir", bin, "--no-start"} + helper := filepath.Join(bin, "agentflow-runner-linux-"+tc.arch) + statePath := filepath.Join(fixtureHome, ".agentflow-runner", "state.json") + const state = `{"runnerId":"fixture"}` + if tc.upgrade { + if err := os.Mkdir(filepath.Dir(statePath), 0o700); err != nil { + t.Fatal(err) + } + for path, data := range map[string]string{ + statePath: state, + filepath.Join(filepath.Dir(statePath), "env"): "AGENTFLOW_SERVER_URL=" + server.URL + "\n", + filepath.Join(bin, "agentflow-runner"): "old host runner", + } { + if err := os.WriteFile(path, []byte(data), 0o700); err != nil { + t.Fatal(err) + } + } + args = append(args, "--upgrade") + } else { + args = append(args, "--server", server.URL, "--token", "fixture") + } + if tc.missingArtifact { + if err := os.WriteFile(helper, []byte("old Linux helper"), 0o755); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command("/bin/bash", args...) + cmd.Env = []string{"HOME=" + fixtureHome, "PATH=" + bin + ":" + filepath.Dir(curl) + ":/usr/bin:/bin", "AGENTFLOW_BROWSER=off"} + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("install: %v\n%s", err, out) + } + assertArtifact := func(path, want string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || string(got) != want { + t.Fatalf("%s = %q, %v; want %q", filepath.Base(path), got, err, want) + } + if info, err := os.Stat(path); err != nil || info.Mode().Perm()&0o100 == 0 { + t.Fatalf("%s is not executable: %v", path, err) + } + } + assertArtifact(filepath.Join(bin, "agentflow-runner"), "runner "+tc.hostOS+"/"+tc.arch) + switch { + case tc.hostOS == "linux": + if _, err := os.Stat(helper); !os.IsNotExist(err) { + t.Fatalf("Linux install wrote a duplicate runner: %v", err) + } + case tc.missingArtifact: + assertArtifact(helper, "old Linux helper") + default: + assertArtifact(helper, "runner linux/"+tc.arch) + assertArtifact(filepath.Join(bin, "agentflow-gate-mcp-linux-"+tc.arch), "gate linux/"+tc.arch) + } + if tc.upgrade { + if got, err := os.ReadFile(statePath); err != nil || string(got) != state { + t.Fatalf("upgrade changed runner state: %q, %v", got, err) + } + } + }) + } +} diff --git a/docs/deployment.md b/docs/deployment.md index 0d1c79995..218893119 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -133,6 +133,8 @@ The upstream process roles remain `-role api`, `-role worker` and `-role all`. A Personal runners are connected from the execution-machine page with a short-lived binding code. The installer is served at `/runners/install.sh`. `AGENTFLOW_RUNNER_BINARY` can point to binaries with `{os}` and `{arch}` substitutions. Container images bundle Linux/macOS amd64/arm64 runner and gate binaries. Native source installations can build on demand with Go. +The macOS installer places the matching Linux runner next to the macOS executable. Docker workspaces use it for Git credentials and MCP calls. When platform updates are enabled, macOS runners also synchronize this helper at startup and retry if it is missing. + A platform administrator can share one CodeBuddy API Key under **Platform administration → Shared CodeBuddy Key** (平台共享 CodeBuddy Key): pick the network environment that issued it and the models members may use (the first is the default). Members without a key of their own then chat and run SOPs on it, limited to those models; a role configured with another model runs on the default and the run says so. Members with their own key keep using it with any model. Scheduled tasks, clarification, reports, bots and run evaluation still need the runner's own key. A scheduled task uses the key for its selected CLI and does not borrow the shared CodeBuddy key. The key is stored in the database like personal keys and is never shown again after saving. `AGENTFLOW_ASSISTANT_SPONSOR_LOGIN`, which lent one account's key limited to `deepseek-v4.1-flash`, still applies while no shared key is configured. Configure provider credentials in the platform UI, not the host's global CLI configuration. Credentials are bound to the responsible actor by the upstream runtime. The home composer and SOP roles each choose a built-in CLI independently. Select the executor beside the home model selector; its API key (and optional Codex/Claude base URL) comes from your personal settings. Changing the executor preserves the draft; saved conversations record the executor used. Main Bot writes still pass through the platform tool gateway and its approval checks. Requirement clarification and notifications continue to use CodeBuddy. A scheduled assistant task can select any supported CLI and runs as an ordinary chat turn; Codex and Claude Code still need that account's own credential. Live model tests are opt-in; set `AGENTFLOW_LIVE_TESTS=1` only for the acceptance suites that need to load local credentials. diff --git a/internal/runner/daemon.go b/internal/runner/daemon.go index f3c8c1696..c1c035288 100644 --- a/internal/runner/daemon.go +++ b/internal/runner/daemon.go @@ -153,6 +153,7 @@ func (d *Daemon) Run(ctx context.Context) error { if err := d.ensureRegistered(ctx); err != nil { return err } + d.refreshDockerRunnerHelper(ctx, false) d.refreshRuntimes(ctx) claimCtx, cancelClaim := context.WithCancel(ctx) defer cancelClaim() @@ -380,6 +381,7 @@ func (d *Daemon) rescanLoop(ctx context.Context) { return case <-ticker.C: d.refreshRuntimes(ctx) + d.refreshDockerRunnerHelper(ctx, true) d.reapIdleRunSandboxes(ctx, runSandboxIdleTTL) } } diff --git a/internal/runner/update.go b/internal/runner/update.go index fddfee923..d14784bfb 100644 --- a/internal/runner/update.go +++ b/internal/runner/update.go @@ -16,6 +16,8 @@ import ( "strings" "sync/atomic" "time" + + flowruntime "agentflow/internal/runtime" ) // ErrUpdateReady tells the entry point to exec the newly installed binary. @@ -51,14 +53,40 @@ func (d *Daemon) checkGateMCPUpdates(ctx context.Context) error { for _, targetOS := range targetOSes { path := strings.ReplaceAll(strings.ReplaceAll(pattern, "{os}", targetOS), "{arch}", runtime.GOARCH) endpoint := fmt.Sprintf("%s/runners/agentflow-gate-mcp?os=%s&arch=%s", d.client.base, targetOS, runtime.GOARCH) - if err := d.checkGateMCPUpdate(ctx, endpoint, path); err != nil { + if err := d.checkCompanionBinaryUpdate(ctx, "gate MCP", endpoint, path); err != nil { return fmt.Errorf("%s/%s: %w", targetOS, runtime.GOARCH, err) } } return nil } -func (d *Daemon) checkGateMCPUpdate(ctx context.Context, endpoint, path string) error { +// Sync at startup after Runner updates, and retry missing artifacts during +// rescans. Conversation workspace requests do not go through the SOP claim loop. +func (d *Daemon) refreshDockerRunnerHelper(ctx context.Context, onlyMissing bool) { + if runtime.GOOS != "darwin" || d.cfg.DisableAutoUpdate || d.runnerWorkActive() { + return + } + updateCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := d.checkDockerRunnerHelperUpdate(updateCtx, onlyMissing); err != nil && ctx.Err() == nil { + log.Printf("Docker Git/MCP helper update: %v", err) + } +} + +func (d *Daemon) checkDockerRunnerHelperUpdate(ctx context.Context, onlyMissing bool) error { + path, err := flowruntime.DockerRunnerHelperPath() + if err != nil { + return err + } + if onlyMissing { + if _, err := os.Stat(path); !os.IsNotExist(err) { + return err + } + } + return d.checkCompanionBinaryUpdate(ctx, "Docker Git/MCP helper", d.runnerUpdateEndpoint("linux"), path) +} + +func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, path string) error { metadata, err := runnerBinaryMetadata(ctx, endpoint) if err != nil { return err @@ -75,9 +103,11 @@ func (d *Daemon) checkGateMCPUpdate(ctx context.Context, endpoint, path string) return err } defer os.Remove(tmp) - d.updateMu.Lock() + if !d.updateMu.TryLock() { + return nil + } defer d.updateMu.Unlock() - if d.updateReady.Load() || atomic.LoadInt32(&d.running) != 0 || ctx.Err() != nil { + if d.updateReady.Load() || d.runnerExecutionActive() || ctx.Err() != nil { return nil } current, err := hashFile(path) @@ -85,12 +115,12 @@ func (d *Daemon) checkGateMCPUpdate(ctx context.Context, endpoint, path string) return err } if current != localHash { - return errors.New("gate MCP executable changed during update") + return fmt.Errorf("%s executable changed during update", name) } if err := os.Rename(tmp, path); err != nil { - return fmt.Errorf("replace gate MCP executable: %w", err) + return fmt.Errorf("replace %s executable: %w", name, err) } - log.Printf("gate MCP update installed (%s) at %s", metadata.SHA256[:12], path) + log.Printf("%s update installed (%s) at %s", name, metadata.SHA256[:12], path) return nil } @@ -107,8 +137,8 @@ func currentRunnerExecutable() (string, error) { var errRunnerUpdateBusy = errors.New("runner is busy") -func (d *Daemon) runnerUpdateEndpoint() string { - return fmt.Sprintf("%s/runners/agentflow-runner?os=%s&arch=%s", d.client.base, runtime.GOOS, runtime.GOARCH) +func (d *Daemon) runnerUpdateEndpoint(targetOS string) string { + return fmt.Sprintf("%s/runners/agentflow-runner?os=%s&arch=%s", d.client.base, targetOS, runtime.GOARCH) } func (d *Daemon) runnerUpdateHashes(ctx context.Context, binary string) (local, expected string, err error) { @@ -116,7 +146,7 @@ func (d *Daemon) runnerUpdateHashes(ctx context.Context, binary string) (local, if err != nil { return "", "", err } - remote, err := runnerBinaryMetadata(ctx, d.runnerUpdateEndpoint()) + remote, err := runnerBinaryMetadata(ctx, d.runnerUpdateEndpoint(runtime.GOOS)) return local, remote.SHA256, err } @@ -157,7 +187,7 @@ func (d *Daemon) installRunnerUpdate(ctx context.Context, binary string) (bool, if strings.EqualFold(localHash, expectedHash) { return false, nil } - tmp, err := downloadRunnerBinary(ctx, d.runnerUpdateEndpoint(), binary, expectedHash) + tmp, err := downloadRunnerBinary(ctx, d.runnerUpdateEndpoint(runtime.GOOS), binary, expectedHash) if err != nil { return false, err } diff --git a/internal/runner/update_test.go b/internal/runner/update_test.go index 6fe88879d..7a37a1a94 100644 --- a/internal/runner/update_test.go +++ b/internal/runner/update_test.go @@ -163,6 +163,70 @@ func TestGateMCPHelperUpdatesWhileRunnerIdle(t *testing.T) { } } +func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { + const payload = "Linux runner for the Docker Git/MCP helper" + var downloads atomic.Int32 + var damaged atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + query := req.URL.Query() + if req.URL.Path != "/runners/agentflow-runner" || query.Get("os") != "linux" || query.Get("arch") != runtime.GOARCH { + http.NotFound(w, req) + return + } + if query.Get("metadata") == "1" { + expected := payload + if damaged.Load() { + expected = "new companion" + } + hash := sha256.Sum256([]byte(expected)) + fmt.Fprintf(w, `{"sha256":%q}`, hex.EncodeToString(hash[:])) + return + } + downloads.Add(1) + if damaged.Load() { + fmt.Fprint(w, "partial data") + return + } + fmt.Fprint(w, payload) + })) + defer server.Close() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture"}) + if err != nil { + t.Fatal(err) + } + bin := t.TempDir() + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(bin, "agentflow-runner-{os}-{arch}")) + helper := filepath.Join(bin, "agentflow-runner-linux-"+runtime.GOARCH) + ctx := context.Background() + if err := d.checkDockerRunnerHelperUpdate(ctx, true); err != nil { + t.Fatal(err) + } + assertHelper := func() { + t.Helper() + if got, err := os.ReadFile(helper); err != nil || string(got) != payload { + t.Fatalf("Docker helper = %q, %v", got, err) + } + if info, err := os.Stat(helper); err != nil || info.Mode().Perm()&0o100 == 0 { + t.Fatalf("helper is not executable: %v", err) + } + if d.updateReady.Load() { + t.Fatal("updating the companion must not restart the runner") + } + } + assertHelper() + if err := d.checkDockerRunnerHelperUpdate(ctx, false); err != nil { + t.Fatal(err) + } + if n := downloads.Load(); n != 1 { + t.Fatalf("already-current helper was re-downloaded, requests = %d", n) + } + damaged.Store(true) + if err := d.checkDockerRunnerHelperUpdate(ctx, false); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("replacement was not rejected: %v", err) + } + assertHelper() +} + func TestRunnerUpdateDefersWhileWorkspaceRequestIsActive(t *testing.T) { const updated = "new runner" hash := sha256.Sum256([]byte(updated)) diff --git a/internal/runtime/mcp_call.go b/internal/runtime/mcp_call.go index 788151704..a6b92e8e9 100644 --- a/internal/runtime/mcp_call.go +++ b/internal/runtime/mcp_call.go @@ -172,21 +172,30 @@ func installMCPCallHelperForSandbox(sandbox Sandbox, workspace, dest string) err if sandboxDriver(sandbox) != "docker" || goruntime.GOOS == "linux" { return installMCPCallHelper(workspace, dest) } - self, err := os.Executable() + src, err := DockerRunnerHelperPath() if err != nil { return err } - pattern := strings.TrimSpace(os.Getenv("AGENTFLOW_RUNNER_BINARY")) - if !strings.Contains(pattern, "{os}") { - pattern = filepath.Join(filepath.Dir(self), "agentflow-runner-{os}-{arch}") - } - src := strings.NewReplacer("{os}", "linux", "{arch}", goruntime.GOARCH).Replace(pattern) if _, err := os.Stat(src); err != nil { return fmt.Errorf("Docker Git/MCP helper requires a Linux runner binary: set AGENTFLOW_RUNNER_BINARY with {os} and {arch}: %w", err) } return installMCPCallHelperFrom(workspace, src, dest) } +// DockerRunnerHelperPath is the Linux runner artifact used for Git and MCP +// inside Docker on a non-Linux runner. Keep installation and lookup in sync. +func DockerRunnerHelperPath() (string, error) { + pattern := strings.TrimSpace(os.Getenv("AGENTFLOW_RUNNER_BINARY")) + if !strings.Contains(pattern, "{os}") { + self, err := os.Executable() + if err != nil { + return "", err + } + pattern = filepath.Join(filepath.Dir(self), "agentflow-runner-{os}-{arch}") + } + return strings.NewReplacer("{os}", "linux", "{arch}", goruntime.GOARCH).Replace(pattern), nil +} + func installMCPCallHelperFrom(workspace, src, dest string) error { srcInfo, err := os.Stat(src) if err != nil { From 4148580c80049c74b8ef5c519fb48f5eef26cddc Mon Sep 17 00:00:00 2001 From: crispinhu Date: Thu, 19 Nov 2026 23:32:10 +0800 Subject: [PATCH 2/3] fix(runner): safeguard macOS Docker helper updates --- deploy/bind-cli.sh | 28 +++- deploy/bind_cli_docker_runner_test.go | 75 ++++++++-- docs/deployment.md | 4 +- internal/runner/daemon.go | 12 +- internal/runner/run_sandbox.go | 5 + internal/runner/update.go | 99 +++++++++++--- internal/runner/update_test.go | 190 +++++++++++++++++++++++++- 7 files changed, 376 insertions(+), 37 deletions(-) diff --git a/deploy/bind-cli.sh b/deploy/bind-cli.sh index d3bf6b696..4f373e1d4 100755 --- a/deploy/bind-cli.sh +++ b/deploy/bind-cli.sh @@ -596,18 +596,40 @@ install_binary() { publish_runner_binary "$BINARY" } +is_linux_runner_binary() { + local header machine + case "$ARCH" in + amd64) machine="3e00" ;; + arm64) machine="b700" ;; + esac + # Read the ELF64 little-endian header. e_type and e_machine start at bytes 16 and 18. + header="$(LC_ALL=C od -An -v -t x1 -N 64 "$1" | tr -d '[:space:]')" || return 1 + [[ ${#header} -eq 128 && "${header:0:14}" == "7f454c46020101" && + ( "${header:14:2}" == "00" || "${header:14:2}" == "03" ) && + ( "${header:32:4}" == "0200" || "${header:32:4}" == "0300" ) && + "${header:36:4}" == "$machine" ]] +} + install_docker_runner_binary() { local source="${SOURCE_DIR:-$PWD}" + # A custom host runner may have a matching helper managed by its installer. + # A per-OS download URL can replace the helper when it is the selected source. + if [[ -x "$DOCKER_RUNNER_BINARY" && -f "$DOCKER_RUNNER_BINARY" && + ( -n "$LOCAL_BINARY" || + ( "$DOWNLOAD_URL" != *"{os}"* && ( -n "$DOWNLOAD_URL" || -n "$SOURCE_DIR" ) ) ) ]]; then + info "保留自定义 Runner 的 Linux Git/MCP helper:${DOCKER_RUNNER_BINARY}" + return + fi TEMP_BINARY=$(mktemp "${BIN_DIR}/.agentflow-runner.XXXXXX") - if download_runner_binary linux || + if { download_runner_binary linux && is_linux_runner_binary "$TEMP_BINARY"; } || { [[ -d "$source/cmd/agentflow-runner" ]] && command -v go >/dev/null 2>&1 && - build_runner_binary linux "$source" && [[ -s "$TEMP_BINARY" ]]; }; then + build_runner_binary linux "$source" && is_linux_runner_binary "$TEMP_BINARY"; }; then publish_runner_binary "$DOCKER_RUNNER_BINARY" return fi rm -f "$TEMP_BINARY" TEMP_BINARY="" - warn "无法获取 Linux/${ARCH} Git/MCP helper,Docker 工作区需要该文件:${DOCKER_RUNNER_BINARY}" + warn "无法获取有效的 Linux/${ARCH} Git/MCP helper,Docker 工作区需要该文件:${DOCKER_RUNNER_BINARY}" } TEMP_BINARY="" diff --git a/deploy/bind_cli_docker_runner_test.go b/deploy/bind_cli_docker_runner_test.go index 9c3899a1b..6629b2b61 100644 --- a/deploy/bind_cli_docker_runner_test.go +++ b/deploy/bind_cli_docker_runner_test.go @@ -1,6 +1,8 @@ package deploy import ( + "debug/elf" + "encoding/binary" "fmt" "net/http" "net/http/httptest" @@ -17,12 +19,19 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { } for _, tc := range []struct { name, hostOS, arch, unameArch string - upgrade, missingArtifact bool + upgrade bool + helperArtifact, source string + preserveHelper bool }{ {name: "macOS_install", hostOS: "darwin", arch: "arm64", unameArch: "arm64"}, {name: "macOS_upgrade", hostOS: "darwin", arch: "amd64", unameArch: "x86_64", upgrade: true}, {name: "Linux_install", hostOS: "linux", arch: "arm64", unameArch: "aarch64"}, - {name: "macOS_preserves_existing_helper_on_failed_download", hostOS: "darwin", arch: "arm64", unameArch: "arm64", missingArtifact: true}, + {name: "macOS_preserves_helper_on_failed_download", hostOS: "darwin", arch: "arm64", unameArch: "arm64", helperArtifact: "unavailable", preserveHelper: true}, + {name: "macOS_preserves_helper_when_server_serves_macOS_binary_for_Linux", hostOS: "darwin", arch: "arm64", unameArch: "arm64", upgrade: true, helperArtifact: "mach-o", preserveHelper: true}, + {name: "macOS_preserves_helper_when_server_serves_wrong_architecture", hostOS: "darwin", arch: "amd64", unameArch: "x86_64", upgrade: true, helperArtifact: "wrong-arch", preserveHelper: true}, + {name: "macOS_preserves_helper_with_local_runner", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", preserveHelper: true}, + {name: "macOS_preserves_helper_with_host_only_URL", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "host-url", preserveHelper: true}, + {name: "macOS_updates_helper_with_explicit_Linux_URL", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "template"}, } { t.Run(tc.name, func(t *testing.T) { fixtureHome := t.TempDir() @@ -34,6 +43,15 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { if err := os.WriteFile(filepath.Join(bin, "uname"), []byte(uname), 0o755); err != nil { t.Fatal(err) } + linuxRunner := bindFixtureLinuxRunner(tc.arch, "new helper") + oldHelper := bindFixtureLinuxRunner(tc.arch, "old helper") + hostRunner := linuxRunner + if tc.hostOS == "darwin" { + hostRunner = "\xcf\xfa\xed\xfe" + tc.arch + " macOS runner" + } + if tc.source == "from" { + hostRunner = "custom " + hostRunner + } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { targetOS, arch := req.URL.Query().Get("os"), req.URL.Query().Get("arch") if arch != tc.arch || (targetOS != tc.hostOS && targetOS != "linux") { @@ -42,11 +60,20 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { } switch req.URL.Path { case "/runners/agentflow-runner": - if tc.missingArtifact && targetOS == "linux" { - http.NotFound(w, req) + if targetOS == tc.hostOS { + fmt.Fprint(w, hostRunner) return } - fmt.Fprintf(w, "runner %s/%s", targetOS, arch) + switch tc.helperArtifact { + case "unavailable": + http.NotFound(w, req) + case "mach-o": + fmt.Fprint(w, hostRunner) + case "wrong-arch": + fmt.Fprint(w, bindFixtureLinuxRunner("arm64", "new helper")) + default: + fmt.Fprint(w, linuxRunner) + } case "/runners/agentflow-gate-mcp": fmt.Fprintf(w, "gate %s/%s", targetOS, arch) default: @@ -75,8 +102,20 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { } else { args = append(args, "--server", server.URL, "--token", "fixture") } - if tc.missingArtifact { - if err := os.WriteFile(helper, []byte("old Linux helper"), 0o755); err != nil { + switch tc.source { + case "from": + custom := filepath.Join(fixtureHome, "custom-runner") + if err := os.WriteFile(custom, []byte(hostRunner), 0o755); err != nil { + t.Fatal(err) + } + args = append(args, "--from", custom) + case "host-url": + args = append(args, "--download-url", server.URL+"/runners/agentflow-runner?os=darwin&arch={arch}") + case "template": + args = append(args, "--download-url", server.URL+"/runners/agentflow-runner?os={os}&arch={arch}") + } + if tc.upgrade || tc.source != "" || tc.preserveHelper { + if err := os.WriteFile(helper, []byte(oldHelper), 0o755); err != nil { t.Fatal(err) } } @@ -95,16 +134,16 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { t.Fatalf("%s is not executable: %v", path, err) } } - assertArtifact(filepath.Join(bin, "agentflow-runner"), "runner "+tc.hostOS+"/"+tc.arch) + assertArtifact(filepath.Join(bin, "agentflow-runner"), hostRunner) switch { case tc.hostOS == "linux": if _, err := os.Stat(helper); !os.IsNotExist(err) { t.Fatalf("Linux install wrote a duplicate runner: %v", err) } - case tc.missingArtifact: - assertArtifact(helper, "old Linux helper") + case tc.preserveHelper: + assertArtifact(helper, oldHelper) default: - assertArtifact(helper, "runner linux/"+tc.arch) + assertArtifact(helper, linuxRunner) assertArtifact(filepath.Join(bin, "agentflow-gate-mcp-linux-"+tc.arch), "gate linux/"+tc.arch) } if tc.upgrade { @@ -115,3 +154,17 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { }) } } + +func bindFixtureLinuxRunner(arch, payload string) string { + header := make([]byte, 64) + copy(header, "\x7fELF\x02\x01\x01\x00") + machine := elf.EM_X86_64 + if arch == "arm64" { + machine = elf.EM_AARCH64 + } + binary.LittleEndian.PutUint16(header[16:], uint16(elf.ET_EXEC)) + binary.LittleEndian.PutUint16(header[18:], uint16(machine)) + binary.LittleEndian.PutUint32(header[20:], uint32(elf.EV_CURRENT)) + binary.LittleEndian.PutUint16(header[52:], uint16(len(header))) + return string(header) + payload +} diff --git a/docs/deployment.md b/docs/deployment.md index 218893119..65657d3ee 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -133,7 +133,9 @@ The upstream process roles remain `-role api`, `-role worker` and `-role all`. A Personal runners are connected from the execution-machine page with a short-lived binding code. The installer is served at `/runners/install.sh`. `AGENTFLOW_RUNNER_BINARY` can point to binaries with `{os}` and `{arch}` substitutions. Container images bundle Linux/macOS amd64/arm64 runner and gate binaries. Native source installations can build on demand with Go. -The macOS installer places the matching Linux runner next to the macOS executable. Docker workspaces use it for Git credentials and MCP calls. When platform updates are enabled, macOS runners also synchronize this helper at startup and retry if it is missing. +The macOS installer places the matching Linux runner next to the macOS executable. Docker workspaces use it for Git credentials and MCP calls. The installer and updater verify the helper's executable format and CPU architecture before replacing it. Custom host installs keep an existing helper. A download URL with an `{os}` placeholder can supply a replacement when used as the runner's download source. + +When platform updates are enabled, macOS runners synchronize the helper in the background at startup and retry if it is missing or incompatible. The runner can report online and accept work during this check. A Docker workspace that needs a missing helper waits for the initial download attempt. A platform administrator can share one CodeBuddy API Key under **Platform administration → Shared CodeBuddy Key** (平台共享 CodeBuddy Key): pick the network environment that issued it and the models members may use (the first is the default). Members without a key of their own then chat and run SOPs on it, limited to those models; a role configured with another model runs on the default and the run says so. Members with their own key keep using it with any model. Scheduled tasks, clarification, reports, bots and run evaluation still need the runner's own key. A scheduled task uses the key for its selected CLI and does not borrow the shared CodeBuddy key. The key is stored in the database like personal keys and is never shown again after saving. `AGENTFLOW_ASSISTANT_SPONSOR_LOGIN`, which lent one account's key limited to `deepseek-v4.1-flash`, still applies while no shared key is configured. diff --git a/internal/runner/daemon.go b/internal/runner/daemon.go index c1c035288..d25db5ce5 100644 --- a/internal/runner/daemon.go +++ b/internal/runner/daemon.go @@ -48,6 +48,7 @@ type Daemon struct { updateMu sync.RWMutex updateReady atomic.Bool updateDownloadMu sync.Mutex + dockerHelperSyncDone chan struct{} // closed after the initial macOS helper sync cancelClaimLoop context.CancelFunc heartbeatWake chan struct{} maintenance runnerMaintenanceState @@ -153,11 +154,13 @@ func (d *Daemon) Run(ctx context.Context) error { if err := d.ensureRegistered(ctx); err != nil { return err } - d.refreshDockerRunnerHelper(ctx, false) d.refreshRuntimes(ctx) claimCtx, cancelClaim := context.WithCancel(ctx) defer cancelClaim() d.cancelClaimLoop = cancelClaim + if goruntime.GOOS == "darwin" && !d.cfg.DisableAutoUpdate { + d.dockerHelperSyncDone = make(chan struct{}) + } go d.rescanLoop(ctx) heartbeatDone := make(chan struct{}) go func() { defer close(heartbeatDone); d.heartbeatLoop(ctx) }() @@ -173,6 +176,9 @@ func (d *Daemon) Run(ctx context.Context) error { <-controlDone d.maintenance.jobs.Wait() <-heartbeatDone + if d.dockerHelperSyncDone != nil { + <-d.dockerHelperSyncDone + } closeCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) d.closeAllRunSandboxes(closeCtx) cancel() @@ -373,6 +379,10 @@ func (d *Daemon) availableProviders() []string { } func (d *Daemon) rescanLoop(ctx context.Context) { + if d.dockerHelperSyncDone != nil { + d.refreshDockerRunnerHelper(ctx, false) + close(d.dockerHelperSyncDone) + } ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() for { diff --git a/internal/runner/run_sandbox.go b/internal/runner/run_sandbox.go index 9cd04f29a..0013ad52b 100644 --- a/internal/runner/run_sandbox.go +++ b/internal/runner/run_sandbox.go @@ -38,6 +38,11 @@ func (d *Daemon) acquireTaskSandbox(ctx context.Context, task domain.RunnerTask) if task.Sandbox == nil { return nil, nil, fmt.Errorf("frozen run sandbox definition is required") } + if strings.TrimSpace(task.Sandbox.Profile.Driver) == domain.SandboxDriverDocker { + if err := d.waitForDockerRunnerHelper(ctx); err != nil { + return nil, nil, err + } + } requirements, resources := taskSandboxSpec(task) key, id, shared := runSandboxKey(task, requirements, resources) if !shared || d.cfg.DisableRunSandboxReuse { diff --git a/internal/runner/update.go b/internal/runner/update.go index d14784bfb..f4f87d9c7 100644 --- a/internal/runner/update.go +++ b/internal/runner/update.go @@ -3,6 +3,7 @@ package runner import ( "context" "crypto/sha256" + "debug/elf" "encoding/hex" "encoding/json" "errors" @@ -53,40 +54,91 @@ func (d *Daemon) checkGateMCPUpdates(ctx context.Context) error { for _, targetOS := range targetOSes { path := strings.ReplaceAll(strings.ReplaceAll(pattern, "{os}", targetOS), "{arch}", runtime.GOARCH) endpoint := fmt.Sprintf("%s/runners/agentflow-gate-mcp?os=%s&arch=%s", d.client.base, targetOS, runtime.GOARCH) - if err := d.checkCompanionBinaryUpdate(ctx, "gate MCP", endpoint, path); err != nil { + if err := d.checkCompanionBinaryUpdate(ctx, "gate MCP", endpoint, path, nil, true); err != nil { return fmt.Errorf("%s/%s: %w", targetOS, runtime.GOARCH, err) } } return nil } -// Sync at startup after Runner updates, and retry missing artifacts during -// rescans. Conversation workspace requests do not go through the SOP claim loop. -func (d *Daemon) refreshDockerRunnerHelper(ctx context.Context, onlyMissing bool) { - if runtime.GOOS != "darwin" || d.cfg.DisableAutoUpdate || d.runnerWorkActive() { +// Sync in the background on startup, then retry missing or incompatible +// helpers during rescans. Conversation requests bypass the SOP claim loop. +func (d *Daemon) refreshDockerRunnerHelper(ctx context.Context, repairOnly bool) { + if runtime.GOOS != "darwin" || d.cfg.DisableAutoUpdate { return } updateCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if err := d.checkDockerRunnerHelperUpdate(updateCtx, onlyMissing); err != nil && ctx.Err() == nil { + if err := d.checkDockerRunnerHelperUpdate(updateCtx, repairOnly); err != nil && ctx.Err() == nil { log.Printf("Docker Git/MCP helper update: %v", err) } } -func (d *Daemon) checkDockerRunnerHelperUpdate(ctx context.Context, onlyMissing bool) error { +func (d *Daemon) checkDockerRunnerHelperUpdate(ctx context.Context, repairOnly bool) error { path, err := flowruntime.DockerRunnerHelperPath() if err != nil { return err } - if onlyMissing { - if _, err := os.Stat(path); !os.IsNotExist(err) { - return err - } + if repairOnly && validateDockerRunnerHelper(path) == nil { + return nil + } + // Sandboxes copy this executable into the workspace, so an atomic replacement + // can run alongside active tasks and requests waiting for the first download. + return d.checkCompanionBinaryUpdate(ctx, "Docker Git/MCP helper", d.runnerUpdateEndpoint("linux"), path, validateDockerRunnerHelper, false) +} + +func (d *Daemon) waitForDockerRunnerHelper(ctx context.Context) error { + if d.dockerHelperSyncDone == nil { + return nil + } + select { + case <-d.dockerHelperSyncDone: + return nil + default: + } + if path, err := flowruntime.DockerRunnerHelperPath(); err == nil && validateDockerRunnerHelper(path) == nil { + return nil + } + select { + case <-d.dockerHelperSyncDone: + return nil + case <-ctx.Done(): + return ctx.Err() } - return d.checkCompanionBinaryUpdate(ctx, "Docker Git/MCP helper", d.runnerUpdateEndpoint("linux"), path) } -func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, path string) error { +func validateDockerRunnerHelper(path string) error { + var machine elf.Machine + switch runtime.GOARCH { + case "arm64": + machine = elf.EM_AARCH64 + case "amd64": + machine = elf.EM_X86_64 + default: + return fmt.Errorf("unsupported Linux helper architecture %s", runtime.GOARCH) + } + info, err := os.Stat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("expected executable Linux/%s helper", runtime.GOARCH) + } + file, err := elf.Open(path) + if err != nil { + return fmt.Errorf("expected Linux/%s ELF helper: %w", runtime.GOARCH, err) + } + defer file.Close() + if file.Class != elf.ELFCLASS64 || file.Data != elf.ELFDATA2LSB || file.Machine != machine || + (file.OSABI != elf.ELFOSABI_NONE && file.OSABI != elf.ELFOSABI_LINUX) || + (file.Type != elf.ET_EXEC && file.Type != elf.ET_DYN) { + return fmt.Errorf("expected Linux/%s executable, got ELF %s %s %s %s %s", runtime.GOARCH, + file.Class, file.Data, file.Machine, file.OSABI, file.Type) + } + return nil +} + +func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, path string, validate func(string) error, requireIdle bool) error { metadata, err := runnerBinaryMetadata(ctx, endpoint) if err != nil { return err @@ -96,6 +148,9 @@ func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, return err } if localHash == metadata.SHA256 { + if validate != nil { + return validate(path) + } return nil } tmp, err := downloadRunnerBinary(ctx, endpoint, path, metadata.SHA256) @@ -103,11 +158,21 @@ func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, return err } defer os.Remove(tmp) - if !d.updateMu.TryLock() { - return nil + if validate != nil { + if err := validate(tmp); err != nil { + return fmt.Errorf("validate %s: %w", name, err) + } } - defer d.updateMu.Unlock() - if d.updateReady.Load() || d.runnerExecutionActive() || ctx.Err() != nil { + if requireIdle { + if !d.updateMu.TryLock() { + return nil + } + defer d.updateMu.Unlock() + if d.runnerExecutionActive() { + return nil + } + } + if d.updateReady.Load() || ctx.Err() != nil { return nil } current, err := hashFile(path) diff --git a/internal/runner/update_test.go b/internal/runner/update_test.go index 7a37a1a94..2df6b7683 100644 --- a/internal/runner/update_test.go +++ b/internal/runner/update_test.go @@ -3,6 +3,8 @@ package runner import ( "context" "crypto/sha256" + "debug/elf" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -16,6 +18,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/gorilla/websocket" ) func TestRunnerUpdateReplacesBinaryOnlyAfterVerification(t *testing.T) { @@ -164,7 +168,7 @@ func TestGateMCPHelperUpdatesWhileRunnerIdle(t *testing.T) { } func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { - const payload = "Linux runner for the Docker Git/MCP helper" + payload := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "Docker helper") var downloads atomic.Int32 var damaged atomic.Bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -198,9 +202,6 @@ func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(bin, "agentflow-runner-{os}-{arch}")) helper := filepath.Join(bin, "agentflow-runner-linux-"+runtime.GOARCH) ctx := context.Background() - if err := d.checkDockerRunnerHelperUpdate(ctx, true); err != nil { - t.Fatal(err) - } assertHelper := func() { t.Helper() if got, err := os.ReadFile(helper); err != nil || string(got) != payload { @@ -213,6 +214,15 @@ func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { t.Fatal("updating the companion must not restart the runner") } } + // A Docker workspace can already be waiting for the helper under the update fence. + d.updateMu.RLock() + atomic.StoreInt32(&d.running, 1) + err = d.checkDockerRunnerHelperUpdate(ctx, true) + atomic.StoreInt32(&d.running, 0) + d.updateMu.RUnlock() + if err != nil { + t.Fatal(err) + } assertHelper() if err := d.checkDockerRunnerHelperUpdate(ctx, false); err != nil { t.Fatal(err) @@ -225,6 +235,178 @@ func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { t.Fatalf("replacement was not rejected: %v", err) } assertHelper() + damaged.Store(false) + if err := os.WriteFile(helper, []byte("invalid existing helper"), 0o755); err != nil { + t.Fatal(err) + } + if err := d.checkDockerRunnerHelperUpdate(ctx, true); err != nil { + t.Fatal(err) + } + assertHelper() +} + +func TestDockerRunnerHelperUpdateRejectsWrongOSOrArchitecture(t *testing.T) { + otherArch := "arm64" + if runtime.GOARCH == otherArch { + otherArch = "amd64" + } + for _, tc := range []struct{ name, payload string }{ + {name: "macOS_binary", payload: "\xcf\xfa\xed\xfemacOS runner"}, + {name: "wrong_Linux_architecture", payload: dockerRunnerFixtureLinuxBinary(otherArch, "wrong architecture")}, + } { + t.Run(tc.name, func(t *testing.T) { + hash := sha256.Sum256([]byte(tc.payload)) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Query().Get("metadata") == "1" { + fmt.Fprintf(w, `{"sha256":%q}`, hex.EncodeToString(hash[:])) + return + } + fmt.Fprint(w, tc.payload) + })) + defer server.Close() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture"}) + if err != nil { + t.Fatal(err) + } + bin := t.TempDir() + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(bin, "agentflow-runner-{os}-{arch}")) + helper := filepath.Join(bin, "agentflow-runner-linux-"+runtime.GOARCH) + original := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "working helper") + if err := os.WriteFile(helper, []byte(original), 0o755); err != nil { + t.Fatal(err) + } + updateErr := d.checkDockerRunnerHelperUpdate(t.Context(), false) + got, err := os.ReadFile(helper) + if err != nil || string(got) != original { + t.Fatalf("incompatible download replaced the helper: %q, read error %v, update error %v", got, err, updateErr) + } + if updateErr == nil || !strings.Contains(updateErr.Error(), "Linux/"+runtime.GOARCH) { + t.Fatalf("expected a platform validation error, got %v", updateErr) + } + }) + } +} + +func TestDockerRunnerHelperDownloadDoesNotBlockMacOSStartup(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("macOS Runner downloads the Linux helper at startup") + } + root := t.TempDir() + t.Setenv("HOME", root) + t.Setenv("PATH", "/usr/bin:/bin") + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(root, "agentflow-runner-{os}-{arch}")) + helper := filepath.Join(root, "agentflow-runner-linux-"+runtime.GOARCH) + original := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "working helper") + if err := os.WriteFile(helper, []byte(original), 0o755); err != nil { + t.Fatal(err) + } + updated := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "updated helper") + hash := sha256.Sum256([]byte(updated)) + helperRequested := make(chan struct{}, 1) + allowMetadata := make(chan struct{}) + heartbeat := make(chan struct{}, 1) + controlHello := make(chan struct{}, 1) + claim := make(chan struct{}, 1) + signal := func(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } + } + upgrader := websocket.Upgrader{} + ctx, cancel := context.WithCancel(t.Context()) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/runners/agentflow-runner": + if req.URL.Query().Get("os") != "linux" || req.URL.Query().Get("arch") != runtime.GOARCH { + http.NotFound(w, req) + return + } + if req.URL.Query().Get("metadata") == "1" { + signal(helperRequested) + select { + case <-allowMetadata: + fmt.Fprintf(w, `{"sha256":%q}`, hex.EncodeToString(hash[:])) + case <-req.Context().Done(): + } + return + } + fmt.Fprint(w, updated) + case "/runners/heartbeat": + signal(heartbeat) + fmt.Fprint(w, `{}`) + case "/runners/connect": + conn, err := upgrader.Upgrade(w, req, nil) + if err != nil { + return + } + defer conn.Close() + var hello controlMessage + if conn.ReadJSON(&hello) == nil && hello.Type == "hello" { + signal(controlHello) + _, _, _ = conn.NextReader() + } + case "/runners/tasks/claim": + signal(claim) + <-ctx.Done() + case "/runners/disconnect": + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, req) + } + })) + defer func() { cancel(); server.Close() }() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture", StatePath: filepath.Join(root, "state.json")}) + if err != nil { + t.Fatal(err) + } + if err := d.saveState(daemonState{RunnerID: "runner", SessionToken: "session", ExpiresAt: time.Now().Add(24 * time.Hour)}); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- d.Run(ctx) }() + defer func() { + cancel() + if err := <-done; err != nil { + t.Errorf("run: %v", err) + } + }() + wait := func(ch <-chan struct{}, event string) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatalf("did not observe %s before the Linux helper download completed", event) + } + } + wait(helperRequested, "Linux helper metadata request") + wait(heartbeat, "heartbeat") + wait(controlHello, "control hello") + wait(claim, "task polling") + workspaceCtx, cancelWorkspace := context.WithTimeout(ctx, time.Second) + defer cancelWorkspace() + if err := d.waitForDockerRunnerHelper(workspaceCtx); err != nil { + t.Fatalf("existing helper should be available during the download: %v", err) + } + close(allowMetadata) + waitFor(t, 5*time.Second, "helper update", func() bool { + got, err := os.ReadFile(helper) + return err == nil && string(got) == updated + }) +} + +func dockerRunnerFixtureLinuxBinary(arch, payload string) string { + header := make([]byte, 64) + copy(header, "\x7fELF\x02\x01\x01\x00") + machine := elf.EM_X86_64 + if arch == "arm64" { + machine = elf.EM_AARCH64 + } + binary.LittleEndian.PutUint16(header[16:], uint16(elf.ET_EXEC)) + binary.LittleEndian.PutUint16(header[18:], uint16(machine)) + binary.LittleEndian.PutUint32(header[20:], uint32(elf.EV_CURRENT)) + binary.LittleEndian.PutUint16(header[52:], uint16(len(header))) + return string(header) + payload } func TestRunnerUpdateDefersWhileWorkspaceRequestIsActive(t *testing.T) { From b85ed4bb5cbce5f21e4d1dbd1a3e8231c837b428 Mon Sep 17 00:00:00 2001 From: crispinhu Date: Fri, 20 Nov 2026 00:17:05 +0800 Subject: [PATCH 3/3] fix(runner): complete Docker helper install and recovery --- deploy/bind-cli.sh | 163 +++++++++++++++--- deploy/bind_cli_docker_runner_test.go | 148 ++++++++++++++-- docs/deployment.md | 6 +- internal/runner/daemon.go | 18 +- internal/runner/run_sandbox.go | 5 - internal/runner/update.go | 62 +++++-- internal/runner/update_test.go | 132 +++++++++++++- internal/runner/workspace.go | 1 + internal/runner/workspace_browser_test.go | 102 +++++++++++ .../runtime/agent_cli_integration_test.go | 2 +- internal/runtime/agent_cli_test.go | 4 +- internal/runtime/agent_cli_zcode_test.go | 2 +- internal/runtime/cli_live_test.go | 2 +- internal/runtime/command_binding.go | 13 +- .../runtime/command_binding_telemetry_test.go | 4 +- .../command_binding_workspace_git_test.go | 4 +- internal/runtime/conversation_process.go | 2 +- internal/runtime/docker_cases_test.go | 2 +- internal/runtime/managed_files_test.go | 2 +- internal/runtime/mcp_call.go | 27 ++- internal/runtime/mcp_call_test.go | 47 ++++- internal/runtime/mcp_stdio_test.go | 2 +- internal/runtime/provider_credentials_test.go | 6 +- .../provider_gateway_integration_test.go | 2 +- .../runtime/run_credentials_extra_test.go | 8 +- internal/runtime/run_git_credentials.go | 2 +- internal/runtime/runtime_test.go | 2 +- internal/runtime/sandbox_local.go | 2 +- internal/runtime/sandbox_manager.go | 2 +- .../runtime/workspace_terminal_sandbox.go | 2 +- 30 files changed, 651 insertions(+), 125 deletions(-) diff --git a/deploy/bind-cli.sh b/deploy/bind-cli.sh index 4f373e1d4..48d0fa3b3 100755 --- a/deploy/bind-cli.sh +++ b/deploy/bind-cli.sh @@ -38,6 +38,7 @@ BIN_DIR="${HOME}/.local/bin" LOCAL_BINARY="" DOWNLOAD_URL="" SOURCE_DIR="" +RUNNER_BUILD_SOURCE="" INSTALL_SERVICE=0 NO_START=0 UNINSTALL=0 @@ -339,11 +340,24 @@ case "$ARCH" in esac [[ "$OS" == "darwin" || "$OS" == "linux" ]] || fail "不支持的系统: ${OS}(Windows 请使用 PowerShell 安装方式)" +# A thin macOS Runner can execute under Rosetta. Its own GOARCH, instead of +# the installer's uname, determines which Linux companion it looks up. +macos_runner_arch() { + local header="" + if [[ -f "$1" ]]; then + header="$(LC_ALL=C od -An -v -t x1 -N 8 "$1" | tr -d '[:space:]')" + fi + case "$header" in + cffaedfe07000001) printf '%s\n' amd64 ;; + cffaedfe0c000001) printf '%s\n' arm64 ;; + *) printf '%s\n' "$ARCH" ;; + esac +} + RUNNER_HOME="${HOME}/.agentflow-runner" ENV_FILE="${RUNNER_HOME}/env" STATE_FILE="${RUNNER_HOME}/state.json" BINARY="${BIN_DIR}/agentflow-runner" -DOCKER_RUNNER_BINARY="${BIN_DIR}/agentflow-runner-linux-${ARCH}" GATE_BINARY_PATTERN="${BIN_DIR}/agentflow-gate-mcp-{os}-{arch}" PLIST="${HOME}/Library/LaunchAgents/com.agentflow.runner.plist" UNIT="${HOME}/.config/systemd/user/agentflow-runner.service" @@ -421,10 +435,11 @@ if [[ "$UNINSTALL" == "1" ]]; then done < <(docker images --format '{{.Repository}}' 2>/dev/null | awk '/^agentflow-tmp-rt-/' || true) info "已清理 runner Docker 沙箱和临时镜像" fi - rm -f "$BINARY" if [[ "$OS" == "darwin" ]]; then - rm -f "$DOCKER_RUNNER_BINARY" + runner_arch=$(macos_runner_arch "$BINARY") + rm -f "${BIN_DIR}/agentflow-runner-linux-${runner_arch}" fi + rm -f "$BINARY" rm -f "${BIN_DIR}"/agentflow-gate-mcp-darwin-amd64 \ "${BIN_DIR}"/agentflow-gate-mcp-darwin-arm64 \ "${BIN_DIR}"/agentflow-gate-mcp-linux-amd64 \ @@ -554,9 +569,10 @@ download_runner_binary() { } build_runner_binary() { - local target_os="$1" source="$2" + local target_os="$1" source="$2" output="$TEMP_BINARY" + [[ "$output" == /* ]] || output="$PWD/$output" info "从源码 ${source} 构建 agentflow-runner(${target_os}/${ARCH})..." - (cd "$source" && CGO_ENABLED=0 GOOS="$target_os" GOARCH="$ARCH" go build -o "$TEMP_BINARY" ./cmd/agentflow-runner) + (cd "$source" && CGO_ENABLED=0 GOOS="$target_os" GOARCH="$ARCH" go build -o "$output" ./cmd/agentflow-runner) } publish_runner_binary() { @@ -581,15 +597,15 @@ install_binary() { else # 平台未配置分发:回退到本地源码构建(--source 或当前目录为源码 checkout)。 AUTO_UPDATE=0 - SRC="${SOURCE_DIR:-$PWD}" - if [[ ! -d "$SRC/cmd/agentflow-runner" ]]; then + RUNNER_BUILD_SOURCE="${SOURCE_DIR:-$PWD}" + if [[ ! -d "$RUNNER_BUILD_SOURCE/cmd/agentflow-runner" ]]; then fail "平台未提供二进制且未找到源码。解决方式任选其一: 1) 平台管理员设置 AGENTFLOW_SOURCE_ROOT(源码目录)或 AGENTFLOW_RUNNER_BINARY(预编译产物)后重启服务 2) 本命令追加 --source (需要本机安装 Go) 3) 本命令追加 --from <已编译的 agentflow-runner 二进制路径>" fi command -v go >/dev/null 2>&1 || fail "本机未安装 Go,无法从源码构建;请使用 --from 提供二进制" - build_runner_binary "$OS" "$SRC" \ + build_runner_binary "$OS" "$RUNNER_BUILD_SOURCE" \ || fail "源码构建失败;请检查 Go 版本(需 1.22+)或改用 --from" fi fi @@ -597,33 +613,90 @@ install_binary() { } is_linux_runner_binary() { - local header machine + local binary="$1" machine size layout offset table_size case "$ARCH" in - amd64) machine="3e00" ;; - arm64) machine="b700" ;; + amd64) machine=62 ;; + arm64) machine=183 ;; esac - # Read the ELF64 little-endian header. e_type and e_machine start at bytes 16 and 18. - header="$(LC_ALL=C od -An -v -t x1 -N 64 "$1" | tr -d '[:space:]')" || return 1 - [[ ${#header} -eq 128 && "${header:0:14}" == "7f454c46020101" && - ( "${header:14:2}" == "00" || "${header:14:2}" == "03" ) && - ( "${header:32:4}" == "0200" || "${header:32:4}" == "0300" ) && - "${header:36:4}" == "$machine" ]] + size=$(wc -c < "$binary") || return 1 + local decode_bytes=' + { for (i = 1; i <= NF; i++) bytes[count++] = $i } + function little_endian(offset, width, i, value) { + for (i = width - 1; i >= 0; i--) value = 256 * value + bytes[offset + i] + return value + }' + # ELF64 starts with 64 bytes, followed by a table of 56-byte program headers. + layout=$(LC_ALL=C od -An -v -t u1 -N 64 "$binary" | + awk -v machine="$machine" -v size="$size" "$decode_bytes"' + END { + if (count != 64 || bytes[0] != 127 || bytes[1] != 69 || + bytes[2] != 76 || bytes[3] != 70 || bytes[4] != 2 || + bytes[5] != 1 || bytes[6] != 1 || (bytes[7] != 0 && bytes[7] != 3)) exit 1 + type = little_endian(16, 2) + offset = little_endian(32, 8) + num = little_endian(56, 2) + if ((type != 2 && type != 3) || little_endian(18, 2) != machine || + little_endian(54, 2) != 56 || num == 0 || num * 56 > 65536 || + offset < 64 || num * 56 > size - offset) exit 1 + printf "%.0f %d\n", offset, num * 56 + }') || return 1 + read -r offset table_size <<< "$layout" + # The header can be valid even when the server only has part of the file. + LC_ALL=C od -An -v -t u1 -j "$offset" -N "$table_size" "$binary" | + awk -v size="$size" -v table_size="$table_size" "$decode_bytes"' + END { + if (count != table_size) exit 1 + for (header = 0; header < count; header += 56) { + type = little_endian(header, 4) + flags = little_endian(header + 4, 4) + offset = little_endian(header + 8, 8) + file_size = little_endian(header + 32, 8) + if (type != 0 && file_size > 0 && (offset > size || file_size > size - offset)) exit 1 + if (type != 1) continue + if (file_size > little_endian(header + 40, 8)) exit 1 + if (file_size > 0 && flags % 2 == 1) executable = 1 + } + if (!executable) exit 1 + }' } -install_docker_runner_binary() { +is_custom_docker_runner_helper() { + [[ -x "$1" && -f "$1" ]] || return 1 + # A custom installer may also provide an executable script wrapper. + [[ "$(LC_ALL=C od -An -v -t x1 -N 2 "$1" | tr -d '[:space:]')" == 2321 ]] || + is_linux_runner_binary "$1" +} + +stage_docker_runner_binary() { local source="${SOURCE_DIR:-$PWD}" + if [[ -n "$RUNNER_BUILD_SOURCE" ]]; then + build_runner_binary linux "$RUNNER_BUILD_SOURCE" && is_linux_runner_binary "$TEMP_BINARY" + return + fi + if download_runner_binary linux && is_linux_runner_binary "$TEMP_BINARY"; then + return 0 + fi + [[ -d "$source/cmd/agentflow-runner" ]] && command -v go >/dev/null 2>&1 && + build_runner_binary linux "$source" && is_linux_runner_binary "$TEMP_BINARY" +} + +install_docker_runner_binary() { # A custom host runner may have a matching helper managed by its installer. # A per-OS download URL can replace the helper when it is the selected source. - if [[ -x "$DOCKER_RUNNER_BINARY" && -f "$DOCKER_RUNNER_BINARY" && - ( -n "$LOCAL_BINARY" || - ( "$DOWNLOAD_URL" != *"{os}"* && ( -n "$DOWNLOAD_URL" || -n "$SOURCE_DIR" ) ) ) ]]; then + if [[ -n "$LOCAL_BINARY" || -n "$RUNNER_BUILD_SOURCE" || + ( "$DOWNLOAD_URL" != *"{os}"* && ( -n "$DOWNLOAD_URL" || -n "$SOURCE_DIR" ) ) ]] && + is_custom_docker_runner_helper "$DOCKER_RUNNER_BINARY"; then info "保留自定义 Runner 的 Linux Git/MCP helper:${DOCKER_RUNNER_BINARY}" return fi - TEMP_BINARY=$(mktemp "${BIN_DIR}/.agentflow-runner.XXXXXX") - if { download_runner_binary linux && is_linux_runner_binary "$TEMP_BINARY"; } || - { [[ -d "$source/cmd/agentflow-runner" ]] && command -v go >/dev/null 2>&1 && - build_runner_binary linux "$source" && is_linux_runner_binary "$TEMP_BINARY"; }; then + local helper_dir + helper_dir=$(dirname "$DOCKER_RUNNER_BINARY") + if [[ -d "$DOCKER_RUNNER_BINARY" || "$DOCKER_RUNNER_BINARY" -ef "$BINARY" ]] || + ! mkdir -p "$helper_dir" || ! TEMP_BINARY=$(mktemp "${helper_dir}/.agentflow-runner.XXXXXX"); then + warn "无法准备 Linux Git/MCP helper 的安装位置:${DOCKER_RUNNER_BINARY}" + return + fi + if stage_docker_runner_binary; then publish_runner_binary "$DOCKER_RUNNER_BINARY" return fi @@ -636,6 +709,28 @@ TEMP_BINARY="" trap '[[ -z "$TEMP_BINARY" ]] || rm -f "$TEMP_BINARY"' EXIT install_binary +configure_macos_docker_runner_helper() { + local pattern="${AGENTFLOW_RUNNER_BINARY:-}" + if [[ "$UPGRADE" == "1" && -f "$PLIST" && -z "${AGENTFLOW_RUNNER_BINARY+x}" ]]; then + pattern=$(/usr/bin/plutil -extract EnvironmentVariables.AGENTFLOW_RUNNER_BINARY raw "$PLIST" 2>/dev/null || true) + fi + # Keep the template expansion in sync with runtime.DockerRunnerHelperPath. + pattern="${pattern#"${pattern%%[![:space:]]*}"}" + pattern="${pattern%"${pattern##*[![:space:]]}"}" + DOCKER_RUNNER_BINARY="${BIN_DIR}/agentflow-runner-linux-${ARCH}" + if [[ "$pattern" == *"{os}"* ]]; then + [[ "$pattern" == /* ]] || pattern="$PWD/$pattern" + DOCKER_RUNNER_BINARY="${pattern//\{os\}/linux}" + DOCKER_RUNNER_BINARY="${DOCKER_RUNNER_BINARY//\{arch\}/$ARCH}" + fi + export AGENTFLOW_RUNNER_BINARY="$pattern" +} + +if [[ "$OS" == "darwin" ]]; then + ARCH=$(macos_runner_arch "$BINARY") + configure_macos_docker_runner_helper +fi + install_gate_binary() { local target_os="$1" target_arch="$2" target target="${GATE_BINARY_PATTERN//\{os\}/$target_os}" @@ -663,6 +758,21 @@ fi if [[ "$UPGRADE" == "1" ]]; then if [[ "$OS" == "darwin" && -f "$PLIST" ]]; then + if [[ "$AUTO_UPDATE" == "0" ]]; then + arg_count=$(/usr/bin/plutil -extract ProgramArguments raw -expect array "$PLIST") + if [[ "$arg_count" -eq 0 ]] || + [[ "$(/usr/bin/plutil -extract "ProgramArguments.$((arg_count - 1))" raw "$PLIST")" != "-auto-update=false" ]]; then + /usr/bin/plutil -insert ProgramArguments -string -auto-update=false -append "$PLIST" + fi + fi + if [[ "$AGENTFLOW_RUNNER_BINARY" == *"{os}"* ]]; then + if ! /usr/bin/plutil -type EnvironmentVariables "$PLIST" >/dev/null 2>&1; then + /usr/bin/plutil -insert EnvironmentVariables -dictionary "$PLIST" + fi + /usr/bin/plutil -replace EnvironmentVariables.AGENTFLOW_RUNNER_BINARY -string "$AGENTFLOW_RUNNER_BINARY" "$PLIST" + elif /usr/bin/plutil -type EnvironmentVariables.AGENTFLOW_RUNNER_BINARY "$PLIST" >/dev/null 2>&1; then + /usr/bin/plutil -remove EnvironmentVariables.AGENTFLOW_RUNNER_BINARY "$PLIST" + fi launchctl unload "$PLIST" launchctl load "$PLIST" info "已原地升级并重启 launchd 服务;机器绑定和工作区保持不变" @@ -687,6 +797,7 @@ AGENTFLOW_RUNNER_BOOTSTRAP_TOKEN=${TOKEN} AGENTFLOW_RUNNER_NAME=${NAME:-$(hostname)} AGENTFLOW_RUNNER_CONCURRENCY=${CONCURRENCY} AGENTFLOW_GATE_MCP_BINARY=${GATE_BINARY_PATTERN} +AGENTFLOW_RUNNER_BINARY=${AGENTFLOW_RUNNER_BINARY:-} EOF chmod 600 "$ENV_FILE" info "配置已写入 ${ENV_FILE}(权限 600)" @@ -720,11 +831,13 @@ EOF # plist 值需要 XML 转义(& 等) ESCAPED_PATH=$(printf '%s' "$USER_PATH" | sed 's/&/\&/g; s//\>/g') ESCAPED_GATE_PATTERN=$(printf '%s' "$GATE_BINARY_PATTERN" | sed 's/&/\&/g; s//\>/g') + ESCAPED_RUNNER_PATTERN=$(printf '%s' "$AGENTFLOW_RUNNER_BINARY" | sed 's/&/\&/g; s//\>/g') cat >> "$PLIST" < EnvironmentVariables PATH${ESCAPED_PATH} AGENTFLOW_GATE_MCP_BINARY${ESCAPED_GATE_PATTERN} + AGENTFLOW_RUNNER_BINARY${ESCAPED_RUNNER_PATTERN} RunAtLoad KeepAlive diff --git a/deploy/bind_cli_docker_runner_test.go b/deploy/bind_cli_docker_runner_test.go index 6629b2b61..7409025c5 100644 --- a/deploy/bind_cli_docker_runner_test.go +++ b/deploy/bind_cli_docker_runner_test.go @@ -3,12 +3,16 @@ package deploy import ( "debug/elf" "encoding/binary" + "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" + "runtime" + "slices" + "strings" "testing" ) @@ -17,23 +21,40 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { if err != nil { t.Skip("curl is required for the runner installer") } + script, err := filepath.Abs("bind-cli.sh") + if err != nil { + t.Fatal(err) + } for _, tc := range []struct { - name, hostOS, arch, unameArch string - upgrade bool - helperArtifact, source string - preserveHelper bool + name, hostOS, arch, unameArch string + upgrade, launchd bool + helperArtifact, source, existing string + preserveHelper bool + templatePath bool }{ {name: "macOS_install", hostOS: "darwin", arch: "arm64", unameArch: "arm64"}, + {name: "macOS_creates_nested_runtime_helper_path", hostOS: "darwin", arch: "arm64", unameArch: "arm64", templatePath: true}, {name: "macOS_upgrade", hostOS: "darwin", arch: "amd64", unameArch: "x86_64", upgrade: true}, {name: "Linux_install", hostOS: "linux", arch: "arm64", unameArch: "aarch64"}, {name: "macOS_preserves_helper_on_failed_download", hostOS: "darwin", arch: "arm64", unameArch: "arm64", helperArtifact: "unavailable", preserveHelper: true}, {name: "macOS_preserves_helper_when_server_serves_macOS_binary_for_Linux", hostOS: "darwin", arch: "arm64", unameArch: "arm64", upgrade: true, helperArtifact: "mach-o", preserveHelper: true}, {name: "macOS_preserves_helper_when_server_serves_wrong_architecture", hostOS: "darwin", arch: "amd64", unameArch: "x86_64", upgrade: true, helperArtifact: "wrong-arch", preserveHelper: true}, + {name: "macOS_preserves_helper_when_only_ELF_header_is_served", hostOS: "darwin", arch: "arm64", unameArch: "arm64", upgrade: true, helperArtifact: "truncated-header", preserveHelper: true}, + {name: "macOS_preserves_helper_when_ELF_segment_is_truncated", hostOS: "darwin", arch: "arm64", unameArch: "arm64", upgrade: true, helperArtifact: "truncated-segment", preserveHelper: true}, {name: "macOS_preserves_helper_with_local_runner", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", preserveHelper: true}, + {name: "macOS_preserves_helper_with_implicit_source_build", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "checkout", preserveHelper: true}, + {name: "macOS_upgrades_launchd_with_custom_runner", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", launchd: true, upgrade: true, preserveHelper: true}, + {name: "macOS_upgrades_launchd_on_template_path", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", launchd: true, upgrade: true, templatePath: true}, + {name: "macOS_installs_companion_for_Intel_runner_on_arm64", hostOS: "darwin", arch: "amd64", unameArch: "arm64", source: "from"}, + {name: "macOS_repairs_incompatible_local_helper", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", existing: "mach-o", upgrade: true}, + {name: "macOS_preserves_custom_helper_script", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "from", existing: "script", preserveHelper: true}, {name: "macOS_preserves_helper_with_host_only_URL", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "host-url", preserveHelper: true}, {name: "macOS_updates_helper_with_explicit_Linux_URL", hostOS: "darwin", arch: "arm64", unameArch: "arm64", source: "template"}, } { t.Run(tc.name, func(t *testing.T) { + if tc.launchd && runtime.GOOS != "darwin" { + t.Skip("requires macOS plutil") + } fixtureHome := t.TempDir() bin := filepath.Join(fixtureHome, "bin") if err := os.Mkdir(bin, 0o755); err != nil { @@ -47,20 +68,34 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { oldHelper := bindFixtureLinuxRunner(tc.arch, "old helper") hostRunner := linuxRunner if tc.hostOS == "darwin" { - hostRunner = "\xcf\xfa\xed\xfe" + tc.arch + " macOS runner" + cpu := "\x0c\x00\x00\x01" + if tc.arch == "amd64" { + cpu = "\x07\x00\x00\x01" + } + hostRunner = "\xcf\xfa\xed\xfe" + cpu + " macOS runner" } if tc.source == "from" { - hostRunner = "custom " + hostRunner + hostRunner += " custom" + } + switch tc.existing { + case "mach-o": + oldHelper = "\xcf\xfa\xed\xfe wrong OS" + case "script": + oldHelper = "#!/bin/sh\nexec /opt/runner \"$@\"\n" } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { targetOS, arch := req.URL.Query().Get("os"), req.URL.Query().Get("arch") - if arch != tc.arch || (targetOS != tc.hostOS && targetOS != "linux") { + if (arch != tc.arch && arch != tc.unameArch) || (targetOS != tc.hostOS && targetOS != "linux") { http.NotFound(w, req) return } switch req.URL.Path { case "/runners/agentflow-runner": if targetOS == tc.hostOS { + if tc.source == "checkout" { + http.NotFound(w, req) + return + } fmt.Fprint(w, hostRunner) return } @@ -71,8 +106,12 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { fmt.Fprint(w, hostRunner) case "wrong-arch": fmt.Fprint(w, bindFixtureLinuxRunner("arm64", "new helper")) + case "truncated-header": + fmt.Fprint(w, linuxRunner[:64]) + case "truncated-segment": + fmt.Fprint(w, linuxRunner[:len(linuxRunner)-1]) default: - fmt.Fprint(w, linuxRunner) + fmt.Fprint(w, bindFixtureLinuxRunner(arch, "new helper")) } case "/runners/agentflow-gate-mcp": fmt.Fprintf(w, "gate %s/%s", targetOS, arch) @@ -81,8 +120,13 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { } })) defer server.Close() - args := []string{"bind-cli.sh", "--bin-dir", bin, "--no-start"} + args := []string{script, "--bin-dir", bin, "--no-start"} helper := filepath.Join(bin, "agentflow-runner-linux-"+tc.arch) + patternEnv := "" + if tc.templatePath { + patternEnv = filepath.Join(fixtureHome, "bundle", "{os}", "{arch}", "agentflow-runner") + helper = filepath.Join(fixtureHome, "bundle", "linux", tc.arch, "agentflow-runner") + } statePath := filepath.Join(fixtureHome, ".agentflow-runner", "state.json") const state = `{"runnerId":"fixture"}` if tc.upgrade { @@ -114,15 +158,72 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { case "template": args = append(args, "--download-url", server.URL+"/runners/agentflow-runner?os={os}&arch={arch}") } - if tc.upgrade || tc.source != "" || tc.preserveHelper { + plist := filepath.Join(fixtureHome, "Library", "LaunchAgents", "com.agentflow.runner.plist") + oldServiceArgs := []string{filepath.Join(bin, "agentflow-runner"), "-labels", "fixture", "-auto-update=true"} + if tc.launchd { + if err := os.MkdirAll(filepath.Dir(plist), 0o700); err != nil { + t.Fatal(err) + } + data := fmt.Sprintf("KeepAliveProgramArguments%s", strings.Join(oldServiceArgs, "")) + if err := os.WriteFile(plist, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + launchctl := "#!/bin/sh\nif [ \"$1\" = load ]; then cp \"$2\" \"$HOME/reloaded.plist\"; fi\n" + if err := os.WriteFile(filepath.Join(bin, "launchctl"), []byte(launchctl), 0o755); err != nil { + t.Fatal(err) + } + } + if (tc.upgrade && !tc.templatePath) || tc.source == "template" || tc.preserveHelper { if err := os.WriteFile(helper, []byte(oldHelper), 0o755); err != nil { t.Fatal(err) } } - cmd := exec.Command("/bin/bash", args...) - cmd.Env = []string{"HOME=" + fixtureHome, "PATH=" + bin + ":" + filepath.Dir(curl) + ":/usr/bin:/bin", "AGENTFLOW_BROWSER=off"} - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("install: %v\n%s", err, out) + if tc.source == "checkout" { + if err := os.MkdirAll(filepath.Join(fixtureHome, "cmd", "agentflow-runner"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixtureHome, "source-host"), []byte(hostRunner), 0o755); err != nil { + t.Fatal(err) + } + goBuild := "#!/bin/sh\n[ \"$GOOS\" = darwin ] && cp \"$HOME/source-host\" \"$3\"\n" + if err := os.WriteFile(filepath.Join(bin, "go"), []byte(goBuild), 0o755); err != nil { + t.Fatal(err) + } + } + install := func() string { + t.Helper() + cmd := exec.Command("/bin/bash", args...) + cmd.Dir = fixtureHome + cmd.Env = []string{"HOME=" + fixtureHome, "PATH=" + bin + ":" + filepath.Dir(curl) + ":/usr/bin:/bin", "AGENTFLOW_BROWSER=off"} + if patternEnv != "" { + cmd.Env = append(cmd.Env, "AGENTFLOW_RUNNER_BINARY="+patternEnv) + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("install: %v\n%s", err, out) + } + return string(out) + } + if output := install(); tc.source == "checkout" && !strings.Contains(output, "-auto-update=false") { + t.Fatalf("local source fallback did not disable automatic updates: %s", output) + } + if tc.launchd { + if tc.templatePath { + // Use the saved service path during another upgrade without a shell override. + if err := os.Remove(helper); err != nil { + t.Fatal(err) + } + patternEnv = "" + } + install() // The same upgrade must keep only one disabling argument. + out, err := exec.Command("/usr/bin/plutil", "-extract", "ProgramArguments", "json", "-o", "-", filepath.Join(fixtureHome, "reloaded.plist")).CombinedOutput() + if err != nil { + t.Fatalf("read reloaded launchd arguments: %v %s", err, out) + } + var got []string + if err := json.Unmarshal(out, &got); err != nil || !slices.Equal(got, append(oldServiceArgs, "-auto-update=false")) { + t.Fatalf("launchd started with %q: %v", got, err) + } } assertArtifact := func(path, want string) { t.Helper() @@ -146,6 +247,11 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { assertArtifact(helper, linuxRunner) assertArtifact(filepath.Join(bin, "agentflow-gate-mcp-linux-"+tc.arch), "gate linux/"+tc.arch) } + if tc.templatePath { + if _, err := os.Stat(filepath.Join(bin, "agentflow-runner-linux-"+tc.arch)); !os.IsNotExist(err) { + t.Fatalf("the installer also wrote the fallback helper path: %v", err) + } + } if tc.upgrade { if got, err := os.ReadFile(statePath); err != nil || string(got) != state { t.Fatalf("upgrade changed runner state: %q, %v", got, err) @@ -156,7 +262,8 @@ func TestBindCLIInstallsDockerRunnerHelper(t *testing.T) { } func bindFixtureLinuxRunner(arch, payload string) string { - header := make([]byte, 64) + const elfHeaderSize, programHeaderSize = 64, 56 + header := make([]byte, elfHeaderSize+programHeaderSize) copy(header, "\x7fELF\x02\x01\x01\x00") machine := elf.EM_X86_64 if arch == "arm64" { @@ -165,6 +272,15 @@ func bindFixtureLinuxRunner(arch, payload string) string { binary.LittleEndian.PutUint16(header[16:], uint16(elf.ET_EXEC)) binary.LittleEndian.PutUint16(header[18:], uint16(machine)) binary.LittleEndian.PutUint32(header[20:], uint32(elf.EV_CURRENT)) - binary.LittleEndian.PutUint16(header[52:], uint16(len(header))) + binary.LittleEndian.PutUint64(header[32:], elfHeaderSize) + binary.LittleEndian.PutUint16(header[52:], elfHeaderSize) + binary.LittleEndian.PutUint16(header[54:], programHeaderSize) + binary.LittleEndian.PutUint16(header[56:], 1) + program := header[elfHeaderSize:] + binary.LittleEndian.PutUint32(program, uint32(elf.PT_LOAD)) + binary.LittleEndian.PutUint32(program[4:], uint32(elf.PF_R|elf.PF_X)) + size := uint64(len(header) + len(payload)) + binary.LittleEndian.PutUint64(program[32:], size) + binary.LittleEndian.PutUint64(program[40:], size) return string(header) + payload } diff --git a/docs/deployment.md b/docs/deployment.md index 65657d3ee..c2027a6b9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -133,9 +133,11 @@ The upstream process roles remain `-role api`, `-role worker` and `-role all`. A Personal runners are connected from the execution-machine page with a short-lived binding code. The installer is served at `/runners/install.sh`. `AGENTFLOW_RUNNER_BINARY` can point to binaries with `{os}` and `{arch}` substitutions. Container images bundle Linux/macOS amd64/arm64 runner and gate binaries. Native source installations can build on demand with Go. -The macOS installer places the matching Linux runner next to the macOS executable. Docker workspaces use it for Git credentials and MCP calls. The installer and updater verify the helper's executable format and CPU architecture before replacing it. Custom host installs keep an existing helper. A download URL with an `{os}` placeholder can supply a replacement when used as the runner's download source. +Docker workspaces on macOS use a Linux Runner executable for Git credentials and MCP calls. The installer chooses the architecture of the installed macOS Runner, including Intel binaries running under Rosetta. It places the helper next to the host binary by default. A Runner-side `AGENTFLOW_RUNNER_BINARY` template with `{os}` selects the installation and runtime path, and the installer carries the template into launchd. -When platform updates are enabled, macOS runners synchronize the helper in the background at startup and retry if it is missing or incompatible. The runner can report online and accept work during this check. A Docker workspace that needs a missing helper waits for the initial download attempt. +The installer and updater validate the Linux executable's format, architecture, and load segments before replacing it. Custom installs keep a compatible existing helper. A Runner built from a local checkout can build its companion from that checkout. A download URL with an `{os}` placeholder can supply both binaries when used as the Runner's download source. + +When platform updates are enabled, macOS runners synchronize the helper in the background at startup. A failed version check is retried during rescans, including when an older helper remains usable. Operations that need a missing helper for Git or MCP can wait up to five seconds for the initial sync. The background download continues within its own transfer timeout. A platform administrator can share one CodeBuddy API Key under **Platform administration → Shared CodeBuddy Key** (平台共享 CodeBuddy Key): pick the network environment that issued it and the models members may use (the first is the default). Members without a key of their own then chat and run SOPs on it, limited to those models; a role configured with another model runs on the default and the run says so. Members with their own key keep using it with any model. Scheduled tasks, clarification, reports, bots and run evaluation still need the runner's own key. A scheduled task uses the key for its selected CLI and does not borrow the shared CodeBuddy key. The key is stored in the database like personal keys and is never shown again after saving. `AGENTFLOW_ASSISTANT_SPONSOR_LOGIN`, which lent one account's key limited to `deepseek-v4.1-flash`, still applies while no shared key is configured. diff --git a/internal/runner/daemon.go b/internal/runner/daemon.go index d25db5ce5..a81144fa9 100644 --- a/internal/runner/daemon.go +++ b/internal/runner/daemon.go @@ -161,7 +161,7 @@ func (d *Daemon) Run(ctx context.Context) error { if goruntime.GOOS == "darwin" && !d.cfg.DisableAutoUpdate { d.dockerHelperSyncDone = make(chan struct{}) } - go d.rescanLoop(ctx) + go d.rescanLoop(ctx, 5*time.Minute) heartbeatDone := make(chan struct{}) go func() { defer close(heartbeatDone); d.heartbeatLoop(ctx) }() controlDone := make(chan struct{}) @@ -378,12 +378,13 @@ func (d *Daemon) availableProviders() []string { return providers } -func (d *Daemon) rescanLoop(ctx context.Context) { +func (d *Daemon) rescanLoop(ctx context.Context, interval time.Duration) { + helperSynced := false if d.dockerHelperSyncDone != nil { - d.refreshDockerRunnerHelper(ctx, false) + helperSynced = d.refreshDockerRunnerHelper(ctx, false) close(d.dockerHelperSyncDone) } - ticker := time.NewTicker(5 * time.Minute) + ticker := time.NewTicker(interval) defer ticker.Stop() for { select { @@ -391,7 +392,9 @@ func (d *Daemon) rescanLoop(ctx context.Context) { return case <-ticker.C: d.refreshRuntimes(ctx) - d.refreshDockerRunnerHelper(ctx, true) + if d.dockerHelperSyncDone != nil { + helperSynced = d.refreshDockerRunnerHelper(ctx, helperSynced) + } d.reapIdleRunSandboxes(ctx, runSandboxIdleTTL) } } @@ -543,6 +546,7 @@ func (d *Daemon) executeTask(ctx context.Context, task domain.RunnerTask) { // locks, cloning repositories, creating sandboxes and collecting outputs too. ctx, stopLease := d.keepTaskLease(ctx, task, leaseRenewInterval) defer stopLease() + ctx = d.withDockerRunnerHelperWait(ctx) // The read fence ends before the completion report: once the control plane // sees this task terminal (and lets a paused run continue), a follow-up // preparation or snapshot must not still find the gate held. @@ -773,7 +777,7 @@ func (d *Daemon) executeTerminal(ctx context.Context, task domain.RunnerTask) ex if err := runtime.ProvisionMCPAssets(ctx, sandbox, workspace, bound, nil); err != nil { return execResult{status: domain.RunnerTaskFailed, errMsg: fmt.Sprintf("provision MCP: %v", err)} } - env, err := runtime.ScriptMCPEnvironment(sandbox, workspace, bound.MCPServers) + env, err := runtime.ScriptMCPEnvironment(ctx, sandbox, workspace, bound.MCPServers) if err != nil { return execResult{status: domain.RunnerTaskFailed, errMsg: fmt.Sprintf("prepare mcp call: %v", err)} } @@ -976,7 +980,7 @@ func (d *Daemon) executeInvocation(ctx context.Context, task domain.RunnerTask) gateAssets, _, _ := runtime.PrepareMCPServers(domain.AgentAssetBundle{MCPServers: servers}, d.mcpTarget(false, inv.GitHTTPCredentials)) servers = gateAssets.MCPServers } - mcpEnv, mcpErr := runtime.ScriptMCPEnvironment(sandbox, workspace, servers) + mcpEnv, mcpErr := runtime.ScriptMCPEnvironment(ctx, sandbox, workspace, servers) if mcpErr != nil { return execResult{status: domain.RunnerTaskFailed, errMsg: fmt.Sprintf("prepare mcp call: %v", mcpErr)}, nil } diff --git a/internal/runner/run_sandbox.go b/internal/runner/run_sandbox.go index 0013ad52b..9cd04f29a 100644 --- a/internal/runner/run_sandbox.go +++ b/internal/runner/run_sandbox.go @@ -38,11 +38,6 @@ func (d *Daemon) acquireTaskSandbox(ctx context.Context, task domain.RunnerTask) if task.Sandbox == nil { return nil, nil, fmt.Errorf("frozen run sandbox definition is required") } - if strings.TrimSpace(task.Sandbox.Profile.Driver) == domain.SandboxDriverDocker { - if err := d.waitForDockerRunnerHelper(ctx); err != nil { - return nil, nil, err - } - } requirements, resources := taskSandboxSpec(task) key, id, shared := runSandboxKey(task, requirements, resources) if !shared || d.cfg.DisableRunSandboxReuse { diff --git a/internal/runner/update.go b/internal/runner/update.go index f4f87d9c7..07e582462 100644 --- a/internal/runner/update.go +++ b/internal/runner/update.go @@ -61,17 +61,16 @@ func (d *Daemon) checkGateMCPUpdates(ctx context.Context) error { return nil } -// Sync in the background on startup, then retry missing or incompatible -// helpers during rescans. Conversation requests bypass the SOP claim loop. -func (d *Daemon) refreshDockerRunnerHelper(ctx context.Context, repairOnly bool) { - if runtime.GOOS != "darwin" || d.cfg.DisableAutoUpdate { - return - } - updateCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - if err := d.checkDockerRunnerHelperUpdate(updateCtx, repairOnly); err != nil && ctx.Err() == nil { - log.Printf("Docker Git/MCP helper update: %v", err) +// Until the first successful sync, a rescan must retry the version check even +// when an older helper is usable. Downloads use the normal updater's timeout. +func (d *Daemon) refreshDockerRunnerHelper(ctx context.Context, repairOnly bool) bool { + if err := d.checkDockerRunnerHelperUpdate(ctx, repairOnly); err != nil { + if ctx.Err() == nil { + log.Printf("Docker Git/MCP helper update: %v", err) + } + return false } + return true } func (d *Daemon) checkDockerRunnerHelperUpdate(ctx context.Context, repairOnly bool) error { @@ -82,28 +81,39 @@ func (d *Daemon) checkDockerRunnerHelperUpdate(ctx context.Context, repairOnly b if repairOnly && validateDockerRunnerHelper(path) == nil { return nil } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create Docker Git/MCP helper directory: %w", err) + } // Sandboxes copy this executable into the workspace, so an atomic replacement // can run alongside active tasks and requests waiting for the first download. return d.checkCompanionBinaryUpdate(ctx, "Docker Git/MCP helper", d.runnerUpdateEndpoint("linux"), path, validateDockerRunnerHelper, false) } -func (d *Daemon) waitForDockerRunnerHelper(ctx context.Context) error { +func (d *Daemon) withDockerRunnerHelperWait(ctx context.Context) context.Context { if d.dockerHelperSyncDone == nil { - return nil + return ctx } + return flowruntime.WithDockerRunnerHelperWait(ctx, d.waitForDockerRunnerHelper) +} + +func (d *Daemon) waitForDockerRunnerHelper(ctx context.Context, path string) error { select { case <-d.dockerHelperSyncDone: return nil default: } - if path, err := flowruntime.DockerRunnerHelperPath(); err == nil && validateDockerRunnerHelper(path) == nil { + if validateDockerRunnerHelper(path) == nil { return nil } + // A workspace request can give up while the download continues. It never + // controls the downloader's deadline. + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() select { case <-d.dockerHelperSyncDone: return nil - case <-ctx.Done(): - return ctx.Err() + case <-waitCtx.Done(): + return fmt.Errorf("Docker Git/MCP helper is still syncing: %w", waitCtx.Err()) } } @@ -135,6 +145,23 @@ func validateDockerRunnerHelper(path string) error { return fmt.Errorf("expected Linux/%s executable, got ELF %s %s %s %s %s", runtime.GOARCH, file.Class, file.Data, file.Machine, file.OSABI, file.Type) } + size := uint64(info.Size()) + hasExecutableSegment := false + for _, segment := range file.Progs { + if segment.Type != elf.PT_NULL && segment.Filesz > 0 && + (segment.Off > size || segment.Filesz > size-segment.Off) { + return fmt.Errorf("incomplete Linux/%s ELF helper segment", runtime.GOARCH) + } + if segment.Type == elf.PT_LOAD { + if segment.Filesz > segment.Memsz { + return fmt.Errorf("invalid Linux/%s ELF helper load segment", runtime.GOARCH) + } + hasExecutableSegment = hasExecutableSegment || segment.Filesz > 0 && segment.Flags&elf.PF_X != 0 + } + } + if !hasExecutableSegment { + return fmt.Errorf("Linux/%s ELF helper has no executable load segment", runtime.GOARCH) + } return nil } @@ -147,10 +174,7 @@ func (d *Daemon) checkCompanionBinaryUpdate(ctx context.Context, name, endpoint, if err != nil && !os.IsNotExist(err) { return err } - if localHash == metadata.SHA256 { - if validate != nil { - return validate(path) - } + if strings.EqualFold(localHash, metadata.SHA256) && (validate == nil || validate(path) == nil) { return nil } tmp, err := downloadRunnerBinary(ctx, endpoint, path, metadata.SHA256) diff --git a/internal/runner/update_test.go b/internal/runner/update_test.go index 2df6b7683..ed3f898b3 100644 --- a/internal/runner/update_test.go +++ b/internal/runner/update_test.go @@ -199,8 +199,8 @@ func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { t.Fatal(err) } bin := t.TempDir() - t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(bin, "agentflow-runner-{os}-{arch}")) - helper := filepath.Join(bin, "agentflow-runner-linux-"+runtime.GOARCH) + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(bin, "{os}", "{arch}", "agentflow-runner")) + helper := filepath.Join(bin, "linux", runtime.GOARCH, "agentflow-runner") ctx := context.Background() assertHelper := func() { t.Helper() @@ -243,16 +243,26 @@ func TestDockerRunnerHelperUpdateRepairsMissingBinary(t *testing.T) { t.Fatal(err) } assertHelper() + if err := os.Chmod(helper, 0o644); err != nil { + t.Fatal(err) + } + if err := d.checkDockerRunnerHelperUpdate(ctx, true); err != nil { + t.Fatal(err) + } + assertHelper() } -func TestDockerRunnerHelperUpdateRejectsWrongOSOrArchitecture(t *testing.T) { +func TestDockerRunnerHelperUpdateRejectsIncompatibleBinary(t *testing.T) { otherArch := "arm64" if runtime.GOARCH == otherArch { otherArch = "amd64" } + complete := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "full program") for _, tc := range []struct{ name, payload string }{ {name: "macOS_binary", payload: "\xcf\xfa\xed\xfemacOS runner"}, {name: "wrong_Linux_architecture", payload: dockerRunnerFixtureLinuxBinary(otherArch, "wrong architecture")}, + {name: "only_ELF_header", payload: complete[:64]}, + {name: "truncated_load_segment", payload: complete[:len(complete)-1]}, } { t.Run(tc.name, func(t *testing.T) { hash := sha256.Sum256([]byte(tc.payload)) @@ -385,7 +395,7 @@ func TestDockerRunnerHelperDownloadDoesNotBlockMacOSStartup(t *testing.T) { wait(claim, "task polling") workspaceCtx, cancelWorkspace := context.WithTimeout(ctx, time.Second) defer cancelWorkspace() - if err := d.waitForDockerRunnerHelper(workspaceCtx); err != nil { + if err := d.waitForDockerRunnerHelper(workspaceCtx, helper); err != nil { t.Fatalf("existing helper should be available during the download: %v", err) } close(allowMetadata) @@ -395,8 +405,109 @@ func TestDockerRunnerHelperDownloadDoesNotBlockMacOSStartup(t *testing.T) { }) } +func TestDockerRunnerHelperRescanRetriesFailedVersionCheck(t *testing.T) { + root := t.TempDir() + t.Setenv("HOME", root) + t.Setenv("PATH", "/usr/bin:/bin") + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(root, "agentflow-runner-{os}-{arch}")) + helper := filepath.Join(root, "agentflow-runner-linux-"+runtime.GOARCH) + if err := os.WriteFile(helper, []byte(dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "old version")), 0o755); err != nil { + t.Fatal(err) + } + updated := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "updated version") + hash := sha256.Sum256([]byte(updated)) + var metadataRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Query().Get("metadata") == "1" { + if metadataRequests.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + fmt.Fprintf(w, `{"sha256":%q}`, hex.EncodeToString(hash[:])) + return + } + fmt.Fprint(w, updated) + })) + defer server.Close() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture"}) + if err != nil { + t.Fatal(err) + } + d.dockerHelperSyncDone = make(chan struct{}) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + defer close(done) + d.rescanLoop(ctx, 50*time.Millisecond) + }() + defer func() { cancel(); <-done }() + waitFor(t, 5*time.Second, "rescan to replace the old helper after a failed startup sync", func() bool { + data, err := os.ReadFile(helper) + return err == nil && string(data) == updated + }) + if got := metadataRequests.Load(); got != 2 { + t.Fatalf("metadata requests = %d, want a failed check and one retry", got) + } +} + +func TestDockerRunnerHelperSlowDownloadOutlivesWorkspaceWait(t *testing.T) { + root := t.TempDir() + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(root, "agentflow-runner-{os}-{arch}")) + helper := filepath.Join(root, "agentflow-runner-linux-"+runtime.GOARCH) + binary := dockerRunnerFixtureLinuxBinary(runtime.GOARCH, "complete slow download") + hash := sha256.Sum256([]byte(binary)) + downloading := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Query().Get("metadata") == "1" { + fmt.Fprintf(w, `{"sha256":%q}`, hex.EncodeToString(hash[:])) + return + } + fmt.Fprint(w, binary[:64]) + w.(http.Flusher).Flush() + close(downloading) + // Keep the transfer active beyond the old shared 30-second deadline. + timer := time.NewTimer(31 * time.Second) + defer timer.Stop() + select { + case <-timer.C: + fmt.Fprint(w, binary[64:]) + case <-req.Context().Done(): + } + })) + defer server.Close() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture"}) + if err != nil { + t.Fatal(err) + } + d.dockerHelperSyncDone = make(chan struct{}) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + defer close(done) + d.rescanLoop(ctx, time.Hour) + }() + defer func() { cancel(); <-done }() + select { + case <-downloading: + case <-time.After(5 * time.Second): + t.Fatal("helper download did not start") + } + if err := d.waitForDockerRunnerHelper(ctx, helper); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("workspace wait = %v, want its own deadline", err) + } + select { + case <-d.dockerHelperSyncDone: + case <-time.After(40 * time.Second): + t.Fatal("helper sync did not complete") + } + if data, err := os.ReadFile(helper); err != nil || string(data) != binary { + t.Fatalf("slow transfer did not publish the verified helper: %q, %v", data, err) + } +} + func dockerRunnerFixtureLinuxBinary(arch, payload string) string { - header := make([]byte, 64) + const elfHeaderSize, programHeaderSize = 64, 56 + header := make([]byte, elfHeaderSize+programHeaderSize) copy(header, "\x7fELF\x02\x01\x01\x00") machine := elf.EM_X86_64 if arch == "arm64" { @@ -405,7 +516,16 @@ func dockerRunnerFixtureLinuxBinary(arch, payload string) string { binary.LittleEndian.PutUint16(header[16:], uint16(elf.ET_EXEC)) binary.LittleEndian.PutUint16(header[18:], uint16(machine)) binary.LittleEndian.PutUint32(header[20:], uint32(elf.EV_CURRENT)) - binary.LittleEndian.PutUint16(header[52:], uint16(len(header))) + binary.LittleEndian.PutUint64(header[32:], elfHeaderSize) + binary.LittleEndian.PutUint16(header[52:], elfHeaderSize) + binary.LittleEndian.PutUint16(header[54:], programHeaderSize) + binary.LittleEndian.PutUint16(header[56:], 1) + program := header[elfHeaderSize:] + binary.LittleEndian.PutUint32(program, uint32(elf.PT_LOAD)) + binary.LittleEndian.PutUint32(program[4:], uint32(elf.PF_R|elf.PF_X)) + size := uint64(len(header) + len(payload)) + binary.LittleEndian.PutUint64(program[32:], size) + binary.LittleEndian.PutUint64(program[40:], size) return string(header) + payload } diff --git a/internal/runner/workspace.go b/internal/runner/workspace.go index 57842f0b2..4889c0fdd 100644 --- a/internal/runner/workspace.go +++ b/internal/runner/workspace.go @@ -120,6 +120,7 @@ func (d *Daemon) readRunnerWorkspace(ctx context.Context, request domain.RunnerW if d.updateReady.Load() { return domain.RunnerWorkspaceResponse{Status: http.StatusConflict, Error: "Runner 即将重启,请稍后重试"} } + ctx = d.withDockerRunnerHelperWait(ctx) if request.Operation == "conversation_process" { return d.conversationProcess(ctx, request) } diff --git a/internal/runner/workspace_browser_test.go b/internal/runner/workspace_browser_test.go index 15b146402..44611c6f6 100644 --- a/internal/runner/workspace_browser_test.go +++ b/internal/runner/workspace_browser_test.go @@ -3,11 +3,17 @@ package runner import ( "context" "encoding/json" + "fmt" "net/http" + "net/http/httptest" + "os" + "path/filepath" "testing" "time" "agentflow/internal/domain" + + "github.com/gorilla/websocket" ) func TestHandleRunnerWorkspaceBrowserRejectsMissingBrowser(t *testing.T) { @@ -52,6 +58,102 @@ func TestHandleRunnerWorkspaceBrowserReportsEnsureFailure(t *testing.T) { } } +func TestDockerBrowserRequestsDoNotWaitForRunnerHelper(t *testing.T) { + root := t.TempDir() + t.Setenv("AGENTFLOW_BROWSER", "always") + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(root, "{os}", "{arch}", "agentflow-runner")) + // Exercise the Docker browser path without launching a real browser. + if err := os.WriteFile(filepath.Join(root, "docker"), []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", root+":/usr/bin:/bin") + + const runID = "browser-helper-sync" + if err := os.MkdirAll(filepath.Join(root, "runs", runID, "workspace"), 0o700); err != nil { + t.Fatal(err) + } + request := domain.RunnerWorkspaceRequest{ + RunID: runID, WorkspaceLayout: domain.RunnerWorkspaceLayoutShared, + Operation: domain.RunnerWorkspaceOperationBrowser, Browser: &domain.BrowserRequest{Ensure: true}, + Sandbox: &domain.RunnerSandboxSpec{ + Profile: domain.SandboxProfile{Driver: domain.SandboxDriverDocker}, + Dockerfile: &domain.DockerfileDefinition{Content: "FROM fixture"}, + }, + Workflow: &domain.WorkflowDefinition{}, + } + upgrader := websocket.Upgrader{} + result := make(chan error, 1) + probe := func(conn *websocket.Conn) error { + if err := conn.SetReadDeadline(time.Now().Add(4 * time.Second)); err != nil { + return err + } + var hello controlMessage + if err := conn.ReadJSON(&hello); err != nil { + return err + } + const slots = 8 + for index := range slots { + if err := conn.WriteJSON(controlMessage{ + Type: "workspace.request", RequestID: fmt.Sprint(index), WorkspaceRequest: &request, + }); err != nil { + return err + } + } + for range slots { + var response controlMessage + if err := conn.ReadJSON(&response); err != nil { + return fmt.Errorf("browser requests waited for the helper: %w", err) + } + if response.WorkspaceResponse == nil || response.WorkspaceResponse.Status != http.StatusOK { + return fmt.Errorf("unexpected browser response: %+v", response.WorkspaceResponse) + } + } + info := domain.RunnerWorkspaceRequest{RunID: runID, WorkspaceLayout: domain.RunnerWorkspaceLayoutShared, Operation: domain.RunnerWorkspaceOperationInfo} + if err := conn.WriteJSON(controlMessage{Type: "workspace.request", RequestID: "info", WorkspaceRequest: &info}); err != nil { + return err + } + var response controlMessage + if err := conn.ReadJSON(&response); err != nil { + return err + } + if response.WorkspaceResponse == nil || response.WorkspaceResponse.Status != http.StatusOK { + return fmt.Errorf("workspace info did not have a free slot: %+v", response.WorkspaceResponse) + } + return nil + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + conn, err := upgrader.Upgrade(w, req, nil) + if err != nil { + result <- err + return + } + defer conn.Close() + result <- probe(conn) + })) + defer server.Close() + d, err := New(Config{ServerURL: server.URL, BootstrapToken: "fixture", DefaultWorkspaceRoot: root}) + if err != nil { + t.Fatal(err) + } + // Model a cold start whose first helper download is still in flight. + d.dockerHelperSyncDone = make(chan struct{}) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + defer close(done) + d.controlLoop(ctx) + }() + defer func() { cancel(); <-done }() + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("control loop did not complete the browser and workspace requests") + } +} + // 只读操作(status/frame)不续期沙箱,写操作续期;HandleBrowser 报错映射为 409。 func TestHandleRunnerWorkspaceBrowserTouchesSandboxOnlyForWrites(t *testing.T) { const runID = "run-browser-touch" diff --git a/internal/runtime/agent_cli_integration_test.go b/internal/runtime/agent_cli_integration_test.go index fb5eb0aff..bd00189c7 100644 --- a/internal/runtime/agent_cli_integration_test.go +++ b/internal/runtime/agent_cli_integration_test.go @@ -23,7 +23,7 @@ type credentialSandbox struct { } func (s credentialSandbox) Exec(ctx context.Context, command CommandSpec, sink EventSink) (string, error) { - bound, err := bindCommand(s.localSandbox, s.workspace, command, s.credentials) + bound, err := bindCommand(ctx, s.localSandbox, s.workspace, command, s.credentials) if err != nil { return "", err } diff --git a/internal/runtime/agent_cli_test.go b/internal/runtime/agent_cli_test.go index c518421da..ff6a1cbb6 100644 --- a/internal/runtime/agent_cli_test.go +++ b/internal/runtime/agent_cli_test.go @@ -396,7 +396,7 @@ func TestBindCommandRejectsSymlinkedParentsBeforeWritingCredentials(t *testing.T credentials := RunCredentials{AgentCLI: map[string]ProviderCredential{ kind: {APIKey: "fake-test-key", BaseURL: "https://example.test"}, }} - if _, err := bindCommand(sb, workspace, command, credentials); err == nil { + if _, err := bindCommand(t.Context(), sb, workspace, command, credentials); err == nil { t.Fatal("symlinked private-home component was accepted") } entries, err := os.ReadDir(outside) @@ -428,7 +428,7 @@ func TestBindAgentCLICommandRejectsSymlinkedPrivateHome(t *testing.T) { if _, _, err := BindAgentCLICommand(sb, workspace, command, ProviderCredential{APIKey: "secret-key", BaseURL: "https://example.test"}, command.Argv); err == nil { t.Fatal("symlinked private home was accepted") } - if _, err := bindCommand(sb, workspace, command, RunCredentials{}); err == nil { + if _, err := bindCommand(t.Context(), sb, workspace, command, RunCredentials{}); err == nil { t.Fatal("bindCommand accepted a symlinked private home") } if entries, _ := os.ReadDir(outside); len(entries) != 0 { diff --git a/internal/runtime/agent_cli_zcode_test.go b/internal/runtime/agent_cli_zcode_test.go index e9c8ace03..49f6e5bea 100644 --- a/internal/runtime/agent_cli_zcode_test.go +++ b/internal/runtime/agent_cli_zcode_test.go @@ -290,7 +290,7 @@ func TestZCodeBindCommandComposition(t *testing.T) { command.CredentialBindings = []CredentialBinding{BindAgentCLI, BindGit, BindMCP} command.PrivateHome = true command.PrivateHomeKey = "zcode-a" - bound, err := bindCommand(sb, workspace, command, RunCredentials{AgentCLI: map[string]ProviderCredential{AgentCLIZCode: {APIKey: "k"}}}) + bound, err := bindCommand(t.Context(), sb, workspace, command, RunCredentials{AgentCLI: map[string]ProviderCredential{AgentCLIZCode: {APIKey: "k"}}}) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/cli_live_test.go b/internal/runtime/cli_live_test.go index b2b6e12c5..c545ab29f 100644 --- a/internal/runtime/cli_live_test.go +++ b/internal/runtime/cli_live_test.go @@ -93,7 +93,7 @@ func runLiveSOP(t *testing.T, root, kind, driver string, item liveCLIConfig) { out = redactLive(out) if err != nil { shown := command.Argv - if bound, berr := bindCommand(sb, workspace, command, creds); berr == nil { + if bound, berr := bindCommand(t.Context(), sb, workspace, command, creds); berr == nil { shown = bound.Argv } t.Fatalf("执行失败: %s\nargv=%q\n%s", redactLive(err.Error()), shown, clipLive(out)) diff --git a/internal/runtime/command_binding.go b/internal/runtime/command_binding.go index fa4f155d3..c67768887 100644 --- a/internal/runtime/command_binding.go +++ b/internal/runtime/command_binding.go @@ -4,6 +4,7 @@ import ( "agentflow/internal/domain" "agentflow/internal/gitwork" "agentflow/internal/observability" + "context" "fmt" "os" "path/filepath" @@ -32,7 +33,7 @@ func upsertEnv(env []string, key, value string) []string { return out } -func bindCommand(sandbox Sandbox, workspace string, command CommandSpec, credentials RunCredentials) (CommandSpec, error) { +func bindCommand(ctx context.Context, sandbox Sandbox, workspace string, command CommandSpec, credentials RunCredentials) (CommandSpec, error) { if command.bindingsPrepared { return command, nil } @@ -114,11 +115,11 @@ func bindCommand(sandbox Sandbox, workspace string, command CommandSpec, credent } bound.environment["CODEBUDDY_INTERNET_ENVIRONMENT"] = strings.TrimSpace(credentials.CodeBuddy.InternetEnvironment) case BindGit: - if err := bindWorkspaceGitEnvironment(sandbox, workspace, bound.environment, credentials); err != nil { + if err := bindWorkspaceGitEnvironment(ctx, sandbox, workspace, bound.environment, credentials); err != nil { return CommandSpec{}, err } case BindGitWorkspace: - if err := bindWorkspaceGitEnvironment(sandbox, workspace, bound.environment, credentials); err != nil { + if err := bindWorkspaceGitEnvironment(ctx, sandbox, workspace, bound.environment, credentials); err != nil { return CommandSpec{}, err } case BindExtra: @@ -130,7 +131,7 @@ func bindCommand(sandbox Sandbox, workspace string, command CommandSpec, credent bound.credentialEnvKeys = append(bound.credentialEnvKeys, envName) } case BindMCP: - if err := bindMCPCallEnvironment(sandbox, workspace, bound.environment, credentials.MCPServers); err != nil { + if err := bindMCPCallEnvironment(ctx, sandbox, workspace, bound.environment, credentials.MCPServers); err != nil { return CommandSpec{}, err } bound.Argv = BindMCPCallArgv(sandbox, bound.Argv) @@ -302,7 +303,7 @@ func privateHomePaths(sandbox Sandbox, workspace string, command CommandSpec) (h // bindWorkspaceGitEnvironment installs the standard host-scoped helper. // It requests credentials through a live relay rather than writing tokens. -func bindWorkspaceGitEnvironment(sandbox Sandbox, workspace string, environment map[string]string, credentials RunCredentials) error { +func bindWorkspaceGitEnvironment(ctx context.Context, sandbox Sandbox, workspace string, environment map[string]string, credentials RunCredentials) error { environment["GIT_TERMINAL_PROMPT"] = "0" if _, err := EnsureControlDirectory(workspace, "runtime"); err != nil { return fmt.Errorf("create git runtime dir: %w", err) @@ -346,7 +347,7 @@ func bindWorkspaceGitEnvironment(sandbox Sandbox, workspace string, environment if _, err := EnsureControlDirectory(workspace, "runtime", "bin"); err != nil { return fmt.Errorf("create git helper dir: %w", err) } - if err := installMCPCallHelperForSandbox(sandbox, workspace, filepath.Join(hostBin, "agentflow-helper")); err != nil { + if err := installMCPCallHelperForSandbox(ctx, sandbox, workspace, filepath.Join(hostBin, "agentflow-helper")); err != nil { return fmt.Errorf("install git credential helper: %w", err) } helper := runtimePath("bin/agentflow-helper") diff --git a/internal/runtime/command_binding_telemetry_test.go b/internal/runtime/command_binding_telemetry_test.go index 48add7aea..bddc9ec27 100644 --- a/internal/runtime/command_binding_telemetry_test.go +++ b/internal/runtime/command_binding_telemetry_test.go @@ -38,14 +38,14 @@ func TestBindTelemetryEnvironmentSetsCollectorAndRunTags(t *testing.T) { func TestBindCommandAppliesTelemetryEnvironment(t *testing.T) { t.Setenv("AGENTFLOW_OTEL_TRACES_ENDPOINT", "http://collector.test/v1/traces") workspace := t.TempDir() - plain, err := bindCommand(&localSandbox{workspace: workspace}, workspace, CommandSpec{Argv: []string{"true"}}, RunCredentials{}) + plain, err := bindCommand(t.Context(), &localSandbox{workspace: workspace}, workspace, CommandSpec{Argv: []string{"true"}}, RunCredentials{}) if err != nil { t.Fatal(err) } if _, ok := plain.environment["AGENTFLOW_RUN_ID"]; ok { t.Fatalf("command without telemetry got telemetry env: %#v", plain.environment) } - traced, err := bindCommand(&localSandbox{workspace: workspace}, workspace, CommandSpec{Argv: []string{"true"}, Telemetry: &InvocationOptions{RunID: "run-2", NodeID: "node-b"}}, RunCredentials{}) + traced, err := bindCommand(t.Context(), &localSandbox{workspace: workspace}, workspace, CommandSpec{Argv: []string{"true"}, Telemetry: &InvocationOptions{RunID: "run-2", NodeID: "node-b"}}, RunCredentials{}) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/command_binding_workspace_git_test.go b/internal/runtime/command_binding_workspace_git_test.go index e1799fd7b..ff62acd35 100644 --- a/internal/runtime/command_binding_workspace_git_test.go +++ b/internal/runtime/command_binding_workspace_git_test.go @@ -129,7 +129,7 @@ func TestBindGitWorkspaceResetsCLIConfig(t *testing.T) { workspace := t.TempDir() sb := &localSandbox{workspace: workspace, driver: "docker"} env := map[string]string{SessionGatewayEnv: "http://relay.invalid/mcp"} - if err := bindWorkspaceGitEnvironment(sb, workspace, env, RunCredentials{GitHTTPHosts: map[string]string{"github.com": "github"}}); err != nil { + if err := bindWorkspaceGitEnvironment(t.Context(), sb, workspace, env, RunCredentials{GitHTTPHosts: map[string]string{"github.com": "github"}}); err != nil { t.Fatal(err) } stale := filepath.Join(workspace, ".agentflow", "runtime", "gh-config", "hosts.yml") @@ -137,7 +137,7 @@ func TestBindGitWorkspaceResetsCLIConfig(t *testing.T) { t.Fatal(err) } env = map[string]string{SessionGatewayEnv: "http://relay.invalid/mcp"} - if err := bindWorkspaceGitEnvironment(sb, workspace, env, RunCredentials{GitHTTPHosts: map[string]string{"github.com": "github"}}); err != nil { + if err := bindWorkspaceGitEnvironment(t.Context(), sb, workspace, env, RunCredentials{GitHTTPHosts: map[string]string{"github.com": "github"}}); err != nil { t.Fatal(err) } if _, err := os.Stat(stale); !os.IsNotExist(err) { diff --git a/internal/runtime/conversation_process.go b/internal/runtime/conversation_process.go index c85c02fdc..53d61ea0f 100644 --- a/internal/runtime/conversation_process.go +++ b/internal/runtime/conversation_process.go @@ -643,7 +643,7 @@ func (p *ConversationProcessPool) start(ctx context.Context, root, workspace str } } nativeArgv := append([]string(nil), command.Argv...) - command, err = bindCommand(bindSandbox, bindWorkspace, command, req.Command.Credentials) + command, err = bindCommand(procCtx, bindSandbox, bindWorkspace, command, req.Command.Credentials) if err != nil { return ConversationProcessResult{}, err } diff --git a/internal/runtime/docker_cases_test.go b/internal/runtime/docker_cases_test.go index fb2ad5643..f390d1a24 100644 --- a/internal/runtime/docker_cases_test.go +++ b/internal/runtime/docker_cases_test.go @@ -44,7 +44,7 @@ func TestDockerSandboxLifecycleWithFakeCLI(t *testing.T) { t.Fatalf("sandbox=%+v", sb) } const extraValue = "test-extra-credential" - command, err := bindCommand(sb, workspace, CommandSpec{ + command, err := bindCommand(t.Context(), sb, workspace, CommandSpec{ Argv: []string{"sh", "-lc", "echo ok"}, CredentialBindings: []CredentialBinding{BindExtra}, ExtraCredentialEnv: map[string]string{"DEPLOYMENT_VALUE": "registry"}, diff --git a/internal/runtime/managed_files_test.go b/internal/runtime/managed_files_test.go index 3a7a63ac0..4ad0ddd9d 100644 --- a/internal/runtime/managed_files_test.go +++ b/internal/runtime/managed_files_test.go @@ -169,7 +169,7 @@ func TestPrepareMCPCallRejectsSymlinks(t *testing.T) { if err := os.Symlink(victim, target); err != nil { t.Fatal(err) } - if _, err := PrepareMCPCall(nil, workspace, nil); err == nil { + if _, err := PrepareMCPCall(t.Context(), nil, workspace, nil); err == nil { t.Fatal("symlink accepted") } if victim != outside { diff --git a/internal/runtime/mcp_call.go b/internal/runtime/mcp_call.go index a6b92e8e9..6483ece31 100644 --- a/internal/runtime/mcp_call.go +++ b/internal/runtime/mcp_call.go @@ -57,7 +57,7 @@ func mcpCallGuestPath(rel string) string { // PrepareMCPCall writes the script-node helper (`agentflow mcp call`) and bound // MCP server list into the workspace. Tokens stay in a 0600 file; they are not // exported as environment variables. -func PrepareMCPCall(sandbox Sandbox, workspace string, servers []domain.MCPServerDefinition) (map[string]string, error) { +func PrepareMCPCall(ctx context.Context, sandbox Sandbox, workspace string, servers []domain.MCPServerDefinition) (map[string]string, error) { hostRoot := mcpCallHostDir(workspace) hostBin := filepath.Join(hostRoot, "bin") if _, err := EnsureControlDirectory(workspace, "runtime", "bin"); err != nil { @@ -80,7 +80,7 @@ func PrepareMCPCall(sandbox Sandbox, workspace string, servers []domain.MCPServe return nil, fmt.Errorf("write mcp call config: %w", err) } helperHost := filepath.Join(hostBin, "agentflow-helper") - if err := installMCPCallHelperForSandbox(sandbox, workspace, helperHost); err != nil { + if err := installMCPCallHelperForSandbox(ctx, sandbox, workspace, helperHost); err != nil { return nil, err } wrapperHost := filepath.Join(hostBin, "agentflow") @@ -168,7 +168,15 @@ func installMCPCallHelper(workspace, dest string) error { return installMCPCallHelperFrom(workspace, src, dest) } -func installMCPCallHelperForSandbox(sandbox Sandbox, workspace, dest string) error { +type dockerRunnerHelperWaitKey struct{} + +// WithDockerRunnerHelperWait lets a personal Runner wait for its background +// download when a command actually needs the Linux Git/MCP executable. +func WithDockerRunnerHelperWait(ctx context.Context, wait func(context.Context, string) error) context.Context { + return context.WithValue(ctx, dockerRunnerHelperWaitKey{}, wait) +} + +func installMCPCallHelperForSandbox(ctx context.Context, sandbox Sandbox, workspace, dest string) error { if sandboxDriver(sandbox) != "docker" || goruntime.GOOS == "linux" { return installMCPCallHelper(workspace, dest) } @@ -176,6 +184,11 @@ func installMCPCallHelperForSandbox(sandbox Sandbox, workspace, dest string) err if err != nil { return err } + if wait, ok := ctx.Value(dockerRunnerHelperWaitKey{}).(func(context.Context, string) error); ok { + if err := wait(ctx, src); err != nil { + return err + } + } if _, err := os.Stat(src); err != nil { return fmt.Errorf("Docker Git/MCP helper requires a Linux runner binary: set AGENTFLOW_RUNNER_BINARY with {os} and {arch}: %w", err) } @@ -254,8 +267,8 @@ func mcpCallAliases(server domain.MCPServerDefinition) []string { return out } -func bindMCPCallEnvironment(sandbox Sandbox, workspace string, environment map[string]string, servers []domain.MCPServerDefinition) error { - prepared, err := PrepareMCPCall(sandbox, workspace, servers) +func bindMCPCallEnvironment(ctx context.Context, sandbox Sandbox, workspace string, environment map[string]string, servers []domain.MCPServerDefinition) error { + prepared, err := PrepareMCPCall(ctx, sandbox, workspace, servers) if err != nil { return err } @@ -344,9 +357,9 @@ func wrapIsolatedMCPCallExec(argv []string) []string { // ScriptMCPEnvironment materializes agentflow mcp call and returns AGENTFLOW_BIN // plus the config path. It does not rewrite PATH for local/unisolated processes. -func ScriptMCPEnvironment(sandbox Sandbox, workspace string, servers []domain.MCPServerDefinition) (map[string]string, error) { +func ScriptMCPEnvironment(ctx context.Context, sandbox Sandbox, workspace string, servers []domain.MCPServerDefinition) (map[string]string, error) { env := map[string]string{} - if err := bindMCPCallEnvironment(sandbox, workspace, env, servers); err != nil { + if err := bindMCPCallEnvironment(ctx, sandbox, workspace, env, servers); err != nil { return nil, err } return env, nil diff --git a/internal/runtime/mcp_call_test.go b/internal/runtime/mcp_call_test.go index 9676c9e6f..42e9b7818 100644 --- a/internal/runtime/mcp_call_test.go +++ b/internal/runtime/mcp_call_test.go @@ -75,7 +75,7 @@ func TestMCPCallCLIInvokesBoundHTTPServer(t *testing.T) { ID: "mcp-2214-project-dns", Name: "Config Center MCP", MarketID: "2214", Transport: "http", URL: gateway.URL, Headers: map[string]string{"Authorization": "Bearer mcp-token"}, }}} - env, err := ScriptMCPEnvironment(sb, workspace, assets.MCPServers) + env, err := ScriptMCPEnvironment(t.Context(), sb, workspace, assets.MCPServers) if err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestMCPCallRequiresPersonalCredentialAtCallTime(t *testing.T) { if err != nil { t.Fatal(err) } - env, err := ScriptMCPEnvironment(sb, workspace, assets.MCPServers) + env, err := ScriptMCPEnvironment(t.Context(), sb, workspace, assets.MCPServers) if err != nil { t.Fatal(err) } @@ -117,7 +117,7 @@ func TestBindMCPWritesWorkspaceBinWithoutPATH(t *testing.T) { workspace := t.TempDir() sb := &localSandbox{workspace: workspace, environment: map[string]string{"PATH": "/usr/bin"}} command := CommandSpec{Argv: []string{"sh", "-lc", "agentflow mcp list"}, CredentialBindings: []CredentialBinding{BindMCP}} - bound, err := bindCommand(sb, workspace, command, RunCredentials{MCPServers: []domain.MCPServerDefinition{{ + bound, err := bindCommand(t.Context(), sb, workspace, command, RunCredentials{MCPServers: []domain.MCPServerDefinition{{ ID: "tapd", Name: "TAPD", Transport: "http", URL: "https://example.com/tapd", }}}) if err != nil { @@ -152,7 +152,7 @@ func TestDockerMCPCallPointsAtWorkspaceBin(t *testing.T) { provideDockerHelperFixture(t) workspace := t.TempDir() sb := &localSandbox{workspace: workspace, driver: "docker"} - env, err := ScriptMCPEnvironment(sb, workspace, []domain.MCPServerDefinition{{ + env, err := ScriptMCPEnvironment(t.Context(), sb, workspace, []domain.MCPServerDefinition{{ ID: "tapd", Name: "TAPD", Transport: "http", URL: "https://example.com/tapd", }}) if err != nil { @@ -198,10 +198,45 @@ func provideDockerHelperFixture(t *testing.T) { t.Setenv("AGENTFLOW_RUNNER_BINARY", pattern) } +func TestDockerHelperBindingWaitsForTheRunnerArtifact(t *testing.T) { + if goruntime.GOOS == "linux" { + t.Skip("a Linux Runner uses its own executable") + } + for _, use := range []string{"git", "mcp"} { + t.Run(use, func(t *testing.T) { + root := t.TempDir() + t.Setenv("AGENTFLOW_RUNNER_BINARY", filepath.Join(root, "runner-{os}-{arch}")) + helper := filepath.Join(root, "runner-linux-"+goruntime.GOARCH) + workspace := t.TempDir() + sb := &localSandbox{workspace: workspace, driver: "docker"} + const payload = "#!/bin/sh\nexit 0\n" + ctx := WithDockerRunnerHelperWait(t.Context(), func(ctx context.Context, path string) error { + if path != helper { + t.Fatalf("waited for %q, want %q", path, helper) + } + return os.WriteFile(path, []byte(payload), 0o755) + }) + var err error + if use == "git" { + env := map[string]string{SessionGatewayEnv: "http://fixture.invalid"} + err = bindWorkspaceGitEnvironment(ctx, sb, workspace, env, RunCredentials{GitHTTPHosts: map[string]string{"git.fixture.invalid": "other"}}) + } else { + _, err = PrepareMCPCall(ctx, sb, workspace, nil) + } + if err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(filepath.Join(mcpCallHostDir(workspace), "bin", "agentflow-helper")); err != nil || string(got) != payload { + t.Fatalf("workspace helper = %q, %v", got, err) + } + }) + } +} + func TestProcessSandboxMCPCallUsesAbsoluteManagedLauncher(t *testing.T) { workspace := t.TempDir() sb := &localSandbox{workspace: workspace, driver: "bwrap"} - env, err := PrepareMCPCall(sb, workspace, []domain.MCPServerDefinition{{ + env, err := PrepareMCPCall(t.Context(), sb, workspace, []domain.MCPServerDefinition{{ ID: "replace-batch", Name: "Replace Batch", Transport: "stdio", Command: "python3", Args: []string{"server.py"}, FileRefs: []domain.MCPFileRef{{Path: "server.py"}}, }}) if err != nil { @@ -263,7 +298,7 @@ func TestHookScriptCanCallMCP(t *testing.T) { } sb := &localSandbox{workspace: workspace} command := CommandSpec{Argv: []string{"sh", "-lc", compiled[0].Command}, CredentialBindings: []CredentialBinding{BindMCP}} - bound, err := bindCommand(sb, workspace, command, RunCredentials{MCPServers: []domain.MCPServerDefinition{{ + bound, err := bindCommand(t.Context(), sb, workspace, command, RunCredentials{MCPServers: []domain.MCPServerDefinition{{ ID: "tapd", Name: "TAPD", MarketID: "70", Transport: "http", URL: "https://example.com/tapd", }}}) if err != nil { diff --git a/internal/runtime/mcp_stdio_test.go b/internal/runtime/mcp_stdio_test.go index b6db2c8ab..cefef34cc 100644 --- a/internal/runtime/mcp_stdio_test.go +++ b/internal/runtime/mcp_stdio_test.go @@ -20,7 +20,7 @@ read initialized read request printf '%s\n' '{"jsonrpc":"2.0","id":2,"result":{"content":[{"type":"text","text":"stdio-ok"}]}}'` servers := []domain.MCPServerDefinition{{ID: "local", Name: "Local", Transport: "stdio", Command: "sh", Args: []string{"-c", serverScript}, TimeoutSec: 5}} - env, err := ScriptMCPEnvironment(sb, workspace, servers) + env, err := ScriptMCPEnvironment(t.Context(), sb, workspace, servers) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/provider_credentials_test.go b/internal/runtime/provider_credentials_test.go index 08de2ce6a..25896c649 100644 --- a/internal/runtime/provider_credentials_test.go +++ b/internal/runtime/provider_credentials_test.go @@ -25,7 +25,7 @@ func TestProviderCredentialBindingIsolation(t *testing.T) { if err != nil { t.Fatal(err) } - bound, err := bindCommand(sb, workspace, command, creds) + bound, err := bindCommand(t.Context(), sb, workspace, command, creds) if err != nil { t.Fatal(err) } @@ -66,13 +66,13 @@ func TestProviderCredentialBindingIsolation(t *testing.T) { t.Fatalf("subprocess environment failed: %v", err) } // Without a responsible user's key, no host credential may be reused. - _, err = bindCommand(sb, workspace, command, RunCredentials{}) + _, err = bindCommand(t.Context(), sb, workspace, command, RunCredentials{}) if err == nil || !strings.Contains(err.Error(), "未配置") { t.Fatal("missing user key must fail before any CLI can reuse host login") } }) } - plain, err := bindCommand(&localSandbox{workspace: t.TempDir()}, t.TempDir(), CommandSpec{Argv: []string{"true"}}, creds) + plain, err := bindCommand(t.Context(), &localSandbox{workspace: t.TempDir()}, t.TempDir(), CommandSpec{Argv: []string{"true"}}, creds) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/provider_gateway_integration_test.go b/internal/runtime/provider_gateway_integration_test.go index 22f579090..6930a5432 100644 --- a/internal/runtime/provider_gateway_integration_test.go +++ b/internal/runtime/provider_gateway_integration_test.go @@ -107,7 +107,7 @@ func TestNativeProviderCustomGateway(t *testing.T) { } creds := RunCredentials{Codex: ProviderCredential{APIKey: "local-gateway-test-key", BaseURL: server.URL + "/v1"}, ClaudeCode: ProviderCredential{APIKey: "local-gateway-test-key", BaseURL: server.URL}} sb := &localSandbox{workspace: workspace} - bound, err := bindCommand(sb, workspace, command, creds) + bound, err := bindCommand(t.Context(), sb, workspace, command, creds) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/run_credentials_extra_test.go b/internal/runtime/run_credentials_extra_test.go index 8c065ed68..31d8f0102 100644 --- a/internal/runtime/run_credentials_extra_test.go +++ b/internal/runtime/run_credentials_extra_test.go @@ -50,7 +50,7 @@ func TestBindExtraCredentialEnv(t *testing.T) { } credentials := RunCredentials{Extra: map[string]string{"gateway": "tok-1"}} - bound, err := bindCommand(sb, sb.workspace, command, credentials) + bound, err := bindCommand(t.Context(), sb, sb.workspace, command, credentials) if err != nil { t.Fatalf("bindCommand: %v", err) } @@ -66,7 +66,7 @@ func TestBindExtraCredentialEnv(t *testing.T) { // Without the binding the mapping is inert. plain := command plain.CredentialBindings = nil - bound, err = bindCommand(sb, sb.workspace, plain, credentials) + bound, err = bindCommand(t.Context(), sb, sb.workspace, plain, credentials) if err != nil { t.Fatalf("bindCommand without binding: %v", err) } @@ -110,7 +110,7 @@ func TestBindExtraMarksCredentialEnvKeysForRedaction(t *testing.T) { CredentialBindings: []CredentialBinding{BindExtra}, ExtraCredentialEnv: map[string]string{"AWS_SECRET_ACCESS_KEY": "gateway"}, } - bound, err := bindCommand(sb, sb.workspace, command, RunCredentials{Extra: map[string]string{"gateway": "tok-1"}}) + bound, err := bindCommand(t.Context(), sb, sb.workspace, command, RunCredentials{Extra: map[string]string{"gateway": "tok-1"}}) if err != nil { t.Fatalf("bindCommand: %v", err) } @@ -152,7 +152,7 @@ func TestConversationPathBindsExtraCredentialEnv(t *testing.T) { CredentialBindings: wire.Bindings, ExtraCredentialEnv: wire.ExtraCredentialEnv, } - bound, err := bindCommand(sb, sb.workspace, rebuilt, wire.Credentials) + bound, err := bindCommand(t.Context(), sb, sb.workspace, rebuilt, wire.Credentials) if err != nil { t.Fatalf("bindCommand: %v", err) } diff --git a/internal/runtime/run_git_credentials.go b/internal/runtime/run_git_credentials.go index 8de21e476..305afa1a0 100644 --- a/internal/runtime/run_git_credentials.go +++ b/internal/runtime/run_git_credentials.go @@ -89,7 +89,7 @@ func RunGitEnvironment(ctx context.Context, sandbox Sandbox, workspace string, c for _, key := range workspaceGitSecretEnvironment { env[key] = "" } - if err := bindWorkspaceGitEnvironment(sandbox, workspace, env, credentials); err != nil { + if err := bindWorkspaceGitEnvironment(ctx, sandbox, workspace, env, credentials); err != nil { closeRelay() return nil, func() {}, err } diff --git a/internal/runtime/runtime_test.go b/internal/runtime/runtime_test.go index c07265979..fa7fe334d 100644 --- a/internal/runtime/runtime_test.go +++ b/internal/runtime/runtime_test.go @@ -819,7 +819,7 @@ func TestLightweightSandboxSmoke(t *testing.T) { t.Fatal(createErr) } allowed := `test "$HOME" != "` + stateHome + `" && test -w "$HOME" && f=$(mktemp) && printf ok > "$f" && printf ok > /dev/null && printf ok > workspace-write.txt` - command, err := bindCommand(sb, workspace, CommandSpec{Argv: []string{fakeCodeBuddy, "-lc", allowed}, Provider: "codebuddy", PrivateHome: true}, RunCredentials{}) + command, err := bindCommand(t.Context(), sb, workspace, CommandSpec{Argv: []string{fakeCodeBuddy, "-lc", allowed}, Provider: "codebuddy", PrivateHome: true}, RunCredentials{}) if err != nil { t.Fatal(err) } diff --git a/internal/runtime/sandbox_local.go b/internal/runtime/sandbox_local.go index aad2bd386..87106831b 100644 --- a/internal/runtime/sandbox_local.go +++ b/internal/runtime/sandbox_local.go @@ -106,7 +106,7 @@ func ExecuteLocalCommand(ctx context.Context, workspace string, command CommandS if cleanup != nil { defer cleanup() } - bound, err := bindCommand(sandbox, workspace, command, credentials) + bound, err := bindCommand(ctx, sandbox, workspace, command, credentials) if err != nil { return "", err } diff --git a/internal/runtime/sandbox_manager.go b/internal/runtime/sandbox_manager.go index b2991d016..65fe2d101 100644 --- a/internal/runtime/sandbox_manager.go +++ b/internal/runtime/sandbox_manager.go @@ -439,7 +439,7 @@ func (s *managedSandbox) Exec(ctx context.Context, command CommandSpec, sink Eve // Binding writes the credential helpers into the workspace; parallel // branches then run their commands concurrently. s.execMu.Lock() - bound, err := bindCommand(s.Sandbox, s.workspace, command, credentials) + bound, err := bindCommand(ctx, s.Sandbox, s.workspace, command, credentials) s.execMu.Unlock() if err != nil { return "", err diff --git a/internal/runtime/workspace_terminal_sandbox.go b/internal/runtime/workspace_terminal_sandbox.go index 1f123fb4d..3c687f821 100644 --- a/internal/runtime/workspace_terminal_sandbox.go +++ b/internal/runtime/workspace_terminal_sandbox.go @@ -117,7 +117,7 @@ func sandboxTerminalCommandBound(ctx context.Context, sandbox Sandbox, command C return nil, err } credentials.GitHTTPHosts = runGitHosts(credentials) - command, err = bindCommand(sandbox, sandbox.Workspace(), prepared, credentials) + command, err = bindCommand(ctx, sandbox, sandbox.Workspace(), prepared, credentials) if err != nil { closeGit() return nil, err