feat: clearer runtime vs CLI updates, default checkpoint path, self-update

README.md

@@ -165,11 +165,18 @@ cosigner-cli logs -f
 | `backup`  | Encrypted backup of the instance (must be stopped)              |
 | `restore` | Restore a backup into an empty directory                        |
 | `export`  | Write `.mshare` files for Fystack Recovery (must be stopped)    |
-| `update`  | Verify and install a newer release                              |
-| `version` | Print CLI and runtime versions                                  |
+| `update`  | Update the **runtime** (container image) to a newer signed release |
+| `self-update` | Update **cosigner-cli** itself to a newer signed release    |
+| `version` | Print CLI and runtime versions (`--check` looks up the newest)  |
+| `help environment` | List every environment variable the CLI reads          |
 
 Global flags: `--home DIR` (or `COSIGNER_HOME`), `--docker CMD`.
 
+For scripts and agents: `version`, `doctor`, `update` and `self-update` take `--json` (the
+document goes to stdout, progress to stderr). `update --check` and `self-update --check` exit
+with status `10` when a newer release exists. Commands never prompt without a terminal: they
+fail, and a `hint:` line names the flag or variable to set (for example `--yes`).
+
 ## Approval callback
 
 `callback add` connects a paired cosigner to your approval callback. It generates the cosigner's
@@ -236,24 +243,27 @@ COSIGNER_PASSWORD_FD=3 cosigner-cli start 3< /run/secrets/cosigner-pw
 | Variable                        | Used by           |
 | ------------------------------- | ----------------- |
 | `COSIGNER_PASSWORD_FD`          | storage password  |
-| `COSIGNER_RECOVERY_PASSWORD_FD` | `backup`/`restore` |
+| `COSIGNER_RECOVERY_PASSWORD_FD` | `backup`/`restore`/`update` |
 
 `COSIGNER_PASSWORD` / `COSIGNER_RECOVERY_PASSWORD` also work but leak into the environment. Avoid them.
 
 ## Backup, export and restore
 
 ```bash
 cosigner-cli stop
-cosigner-cli backup --out /secure/backups/cosigner.enc
+cosigner-cli backup                               # -> ./backups/backup-<version>-<time>.enc
 cosigner-cli export --out-dir /secure/exports     # --list to see wallets, --wallet ID to pick
 cosigner-cli start
 ```
 
 - `backup` asks for a separate **recovery password**. Store the archive and the password in different places.
+- Archives go to `<home>/backups/` unless you pass `--out PATH` or set `COSIGNER_BACKUP_DIR`. Backups never
+  include that folder. A copy on the instance's own disk is lost with the disk, so copy archives off-host.
+  Inside the instance, only `backups/` may hold archives.
 - `export` prints a six-word passphrase once. Store it apart from the `.mshare` files.
 - Backup size limit: 128 MiB of data.
 
-Restore into a new, empty directory:
+Restore into a new, empty directory (it may already contain a `backups/` folder holding the archive):
 
 ```bash
 cosigner-cli --home /srv/cosigner-restored restore --in /secure/backups/cosigner.enc
@@ -265,16 +275,49 @@ cosigner-cli --home /srv/cosigner-restored restore --in /secure/backups/cosigner
 
 ## Update
 
+Two things get updated, each with its own command and version number:
+
+| What | Command | Released from |
+| --- | --- | --- |
+| The **runtime**: the container that holds the key shares and signs | `cosigner-cli update` | `ghcr.io/fystack/cosigner` |
+| **cosigner-cli**: this tool | `cosigner-cli self-update` | GitHub releases of this repository |
+
+`cosigner-cli version --check` shows both, with the newest release of each.
+
+### Runtime
+
 ```bash
-cosigner-cli update --dry-run                                    # preview
-cosigner-cli update --backup-out /secure/backups/pre-update.enc  # install newest
-cosigner-cli update --version v1.2.3 --backup-out ...            # pin a version
+cosigner-cli update --dry-run                                    # verify and preview the newest release
+cosigner-cli update                                              # checkpoint to ./backups/, then install
+cosigner-cli update --backup-out /secure/backups/pre-update.enc  # checkpoint somewhere else
+cosigner-cli update --version v1.2.3 --yes                       # pin a version, no prompt (scripts)
+cosigner-cli update --check                                      # exit 10 when an update exists
 ```
 
-- Requires a newer release, a valid signature and a compatible runtime contract.
-- Always takes an encrypted checkpoint first. There is no automatic rollback.
+- Requires a newer release, a valid signature and a compatible runtime contract. If the runtime
+  needs a newer CLI, `update` says so: run `self-update` first.
+- Shows the plan (installed version, candidate version, checkpoint path, downtime) and asks before
+  it stops anything. Without a terminal, pass `--yes`.
+- Always takes an encrypted checkpoint first, in `<home>/backups/pre-update-<from>-to-<to>-<time>.enc`
+  by default. There is no automatic rollback. Afterwards it prints the restore command for that checkpoint.
+- Signing pauses from the stop until the new runtime is ready.
 - Pulls from the registry the instance was installed from. Switch with `--repository ghcr.io/fystack/cosigner`.
-- `start`, `restart`, `status` and `doctor` tell you when a new release exists. Disable with `COSIGNER_NO_UPDATE_CHECK=1`.
+
+### cosigner-cli
+
+```bash
+cosigner-cli self-update            # newest release
+sudo cosigner-cli self-update       # when the binary is in a root-owned directory such as /usr/local/bin
+cosigner-cli self-update --check    # exit 10 when an update exists
+```
+
+`self-update` downloads the release binary for this platform and verifies it with cosign against
+this repository's release workflow for that tag. That's the same check `install.sh` makes, and it
+runs even with `COSIGNER_SKIP_IMAGE_SIGNATURE_VERIFICATION=1`. The binary is replaced atomically
+only once it reports the expected version. The running Cosigner is not touched.
+
+`start`, `restart`, `status` and `doctor` tell you when a new runtime or CLI release exists, naming
+which one. Disable with `COSIGNER_NO_UPDATE_CHECK=1`.
 
 ## Security notes
 

cmd/cosigner-cli/main.go

@@ -17,7 +17,16 @@ func main() {
 	a := app.New()
 	a.Version, a.Commit = version, commit
 	if err := a.Command().Execute(); err != nil {
+		// Checks such as `update --check` answer with the exit status alone.
+		var exit *app.ExitError
+		if errors.As(err, &exit) {
+			os.Exit(exit.Code)
+		}
 		fmt.Fprintln(os.Stderr, "error:", err)
+		var hinted interface{ Hint() string }
+		if errors.As(err, &hinted) {
+			fmt.Fprintln(os.Stderr, "hint:", hinted.Hint())
+		}
 		var usage *app.UsageError
 		if errors.As(err, &usage) {
 			fmt.Fprint(os.Stderr, usage.Help())

internal/app/app.go

@@ -80,31 +80,44 @@ func (execRunner) Stream(ctx context.Context, name string, args []string, input
 }
 
 type App struct {
-	password        []byte
-	In              io.Reader
-	Out, Err        io.Writer
-	Getenv          func(string) string
-	Runner          Runner
-	Random          io.Reader
-	Tags            func(ctx context.Context, repository string) ([]string, error)
+	password []byte
+	In       io.Reader
+	Out, Err io.Writer
+	Getenv   func(string) string
+	Runner   Runner
+	Random   io.Reader
+	Tags     func(ctx context.Context, repository string) ([]string, error)
+	// LatestCLI returns the newest stable cosigner-cli release tag.
+	LatestCLI func(ctx context.Context) (string, error)
+	// Fetch downloads an HTTPS URL, failing beyond max bytes.
+	Fetch func(ctx context.Context, url string, max int64) ([]byte, error)
+	// Executable is the path of the running cosigner-cli, replaced by self-update.
+	Executable      func() (string, error)
 	Version, Commit string
 }
 
 func New() *App {
-	return &App{In: os.Stdin, Out: os.Stdout, Err: os.Stderr, Getenv: os.Getenv, Runner: execRunner{}, Random: rand.Reader, Tags: registryTags, Version: "dev", Commit: "dev"}
+	return &App{In: os.Stdin, Out: os.Stdout, Err: os.Stderr, Getenv: os.Getenv, Runner: execRunner{}, Random: rand.Reader, Tags: registryTags, LatestCLI: githubLatestRelease, Fetch: httpsFetch, Executable: os.Executable, Version: "dev", Commit: "dev"}
 }
 func (a *App) Command() *cobra.Command {
-	root := &cobra.Command{Use: "cosigner-cli", Short: "Manage a Cosigner installation", SilenceUsage: true, SilenceErrors: true}
+	root := &cobra.Command{Use: "cosigner-cli", Short: "Manage a Cosigner installation", Long: `cosigner-cli installs and runs the Cosigner runtime: the container that holds your
+key shares and signs. The CLI and the runtime are released and versioned separately:
+
+  cosigner-cli version       show both versions (--check looks up the newest)
+  cosigner-cli update        update the runtime (container image)
+  cosigner-cli self-update   update this CLI
+
+Run "cosigner-cli help environment" for the environment variables it reads.`, SilenceUsage: true, SilenceErrors: true}
 	root.SetIn(a.In)
 	root.SetOut(a.Out)
 	root.SetErr(a.Err)
 	root.PersistentFlags().String("home", "", "Instance directory (default: current directory)")
 	root.PersistentFlags().String("docker", "", "Docker command")
-	root.AddCommand(a.setupCommand(), a.startCommand(), a.stopCommand(), a.restartCommand(), a.statusCommand(), a.logsCommand(), a.doctorCommand(), a.infoCommand(), a.backupCommand(), a.restoreCommand(), a.exportCommand(), a.updateCommand(), a.callbackCommand(), a.versionCommand())
+	root.AddCommand(a.setupCommand(), a.startCommand(), a.stopCommand(), a.restartCommand(), a.statusCommand(), a.logsCommand(), a.doctorCommand(), a.infoCommand(), a.backupCommand(), a.restoreCommand(), a.exportCommand(), a.updateCommand(), a.selfUpdateCommand(), a.callbackCommand(), a.versionCommand(), environmentHelp())
 	for _, cmd := range root.Commands() {
 		cmd.Args = cobra.NoArgs
 		switch cmd.Name() {
-		case "version", "logs", "status", "info", "doctor":
+		case "version", "logs", "status", "info", "doctor", "self-update":
 		default:
 			if cmd.RunE != nil {
 				cmd.RunE = a.withInstance(cmd.RunE)

internal/app/app_test.go

@@ -624,16 +624,6 @@ func TestUpdateWithoutVersionUsesNewestRelease(t *testing.T) {
 	require.Empty(t, r.calls)
 }
 
-func TestUpdateRequiresBackupBeforePulling(t *testing.T) {
-	r := &fakeRunner{}
-	a := newTestApp(r)
-	l, _ := fixture(t, a)
-	cmd := a.Command()
-	cmd.SetArgs([]string{"--home", l.home, "update", "--version", "1.2.4"})
-	require.ErrorContains(t, cmd.Execute(), "--backup-out")
-	require.Empty(t, r.calls)
-}
-
 func TestStartupChecksContractWithoutReverifyingSignature(t *testing.T) {
 	r := &fakeRunner{run: func(c runCall) ([]byte, error) {
 		if c.name == "cosign" {

internal/app/commands.go

@@ -612,8 +612,23 @@ func (a *App) logsCommand() *cobra.Command {
 	c.Flags().BoolP("follow", "f", false, "Follow logs")
 	return c
 }
+
+// doctorReport is the JSON shape of `doctor --json`. Failed checks are errors
+// (non-zero exit, message on stderr), not report fields.
+type doctorReport struct {
+	State          string `json:"state"`
+	Profile        string `json:"profile,omitempty"`
+	Image          string `json:"image,omitempty"`
+	RuntimeVersion string `json:"runtime_version,omitempty"`
+	CLIVersion     string `json:"cli_version"`
+}
+
 func (a *App) doctorCommand() *cobra.Command {
-	return &cobra.Command{Use: "doctor", Short: "Check Docker, configuration, and installation state", RunE: func(c *cobra.Command, _ []string) error {
+	c := &cobra.Command{Use: "doctor", Short: "Check Docker, configuration, and installation state", RunE: func(c *cobra.Command, _ []string) error {
+		asJSON, _ := c.Flags().GetBool("json")
+		emit, restore := a.jsonMode(asJSON)
+		defer restore()
+		report := doctorReport{CLIVersion: a.Version}
 		home, err := a.home(c)
 		if err != nil {
 			return err
@@ -629,11 +644,13 @@ func (a *App) doctorCommand() *cobra.Command {
 		}
 		if !fileExists(l.config) {
 			fmt.Fprintln(a.Out, "State: unconfigured")
-			return nil
+			report.State = "unconfigured"
+			return emit(report)
 		}
 		if !fileExists(l.pairing()) {
 			fmt.Fprintln(a.Out, "State: configured but unpaired")
-			return nil
+			report.State = "unpaired"
+			return emit(report)
 		}
 		if err := validatePaired(l, s); err != nil {
 			return err
@@ -655,11 +672,14 @@ func (a *App) doctorCommand() *cobra.Command {
 		if err != nil {
 			return err
 		}
-		fmt.Fprintf(a.Out, "profile: %s\nimage: %s\nruntime_version: %s\n", s.Profile, s.Image, caps.Version)
+		fmt.Fprintf(a.Out, "profile: %s\nimage: %s\nruntime_version: %s\ncli_version: %s\n", s.Profile, s.Image, caps.Version, a.Version)
 		fmt.Fprintln(a.Out, "State: paired (run `cosigner-cli status` for container state)")
 		a.notifyUpdate(c.Context(), s)
-		return nil
+		report.State, report.Profile, report.Image, report.RuntimeVersion = "paired", s.Profile, s.Image, vtag(caps.Version)
+		return emit(report)
 	}}
+	c.Flags().Bool("json", false, "Print the result as JSON on stdout; progress goes to stderr")
+	return c
 }
 func (a *App) infoCommand() *cobra.Command {
 	return &cobra.Command{Use: "info", Short: "Print public Cosigner identity", RunE: func(c *cobra.Command, _ []string) error {
@@ -709,9 +729,3 @@ func (a *App) waitReady(ctx context.Context, docker string, l layout, s Settings
 		}
 	}
 }
-func (a *App) versionCommand() *cobra.Command {
-	return &cobra.Command{Use: "version", Short: "Print host and runtime build identity", RunE: func(*cobra.Command, []string) error {
-		fmt.Fprintf(a.Out, "cosigner-cli %s (commit %s)\n", a.Version, a.Commit)
-		return nil
-	}}
-}

internal/app/cosign.go

@@ -15,6 +15,12 @@ func (a *App) cosignOK(ctx context.Context) error {
 	if a.Getenv("COSIGNER_SKIP_IMAGE_SIGNATURE_VERIFICATION") == "1" {
 		return nil
 	}
+	return a.cosignInstalled(ctx)
+}
+
+// cosignInstalled checks for cosign regardless of the development skip switch;
+// self-update uses it because a CLI binary is never run unverified.
+func (a *App) cosignInstalled(ctx context.Context) error {
 	err := a.step("Checking Cosign", func() (string, error) {
 		raw, err := a.Runner.Run(ctx, "cosign", []string{"version"}, nil, nil)
 		if v := cosignVersion.FindSubmatch(raw); v != nil {

internal/app/hints.go

@@ -0,0 +1,53 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// exitUpdateAvailable is the status of `update --check` and
+// `self-update --check` when a newer release exists, so scripts can branch on it.
+const exitUpdateAvailable = 10
+
+// hintError carries the next command an operator (or an agent) should run.
+// main prints it on its own "hint:" line after the error.
+type hintError struct {
+	err  error
+	hint string
+}
+
+func (e *hintError) Error() string { return e.err.Error() }
+func (e *hintError) Unwrap() error { return e.err }
+func (e *hintError) Hint() string  { return e.hint }
+
+func withHint(err error, format string, args ...any) error {
+	return &hintError{err: err, hint: fmt.Sprintf(format, args...)}
+}
+
+// ExitError ends a command with a status code and no message, for checks whose
+// answer is the exit code.
+type ExitError struct{ Code int }
+
+func (e *ExitError) Error() string { return fmt.Sprintf("exit status %d", e.Code) }
+
+// vtag normalises a release version to the vX.Y.Z form used in all output.
+func vtag(version string) string { return "v" + strings.TrimPrefix(version, "v") }
+
+// jsonMode sends human progress to stderr for the rest of the command, so that
+// stdout carries only the JSON document the returned function writes.
+func (a *App) jsonMode(enabled bool) (emit func(any) error, restore func()) {
+	if !enabled {
+		return func(any) error { return nil }, func() {}
+	}
+	stdout := a.Out
+	a.Out = a.Err
+	return func(v any) error { return writeJSON(stdout, v) }, func() { a.Out = stdout }
+}
+
+func writeJSON(w io.Writer, v any) error {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return enc.Encode(v)
+}

internal/app/password.go

@@ -34,6 +34,10 @@ const (
 	passphraseCheckWords = 3
 )
 
+// errNoPasswordInput means a password was needed but there is no terminal and
+// no environment variable to read it from.
+var errNoPasswordInput = errors.New(passwordEnv + " is required in non-interactive mode")
+
 // encryptedKEK is an encrypted, versioned representation of the hex storage
 // KEK. It deliberately contains KDF parameters and salt, but never a password
 // or derived key.
@@ -87,7 +91,7 @@ func (a *App) readPassword(confirm bool) (result []byte, resultErr error) {
 	}
 	file, ok := a.In.(*os.File)
 	if !ok || !isTerminal(file) {
-		return nil, fmt.Errorf("%s is required in non-interactive mode", passwordEnv)
+		return nil, withHint(errNoPasswordInput, "pass it on an open descriptor, for example COSIGNER_PASSWORD_FD=3 3<passphrase-file, or set %s", passwordEnv)
 	}
 	for {
 		fmt.Fprint(a.Out, "Storage encryption passphrase: ")