From 4c5630f491e7173288fab1ddc4d641e0613bdaab Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 12:55:28 -0400 Subject: [PATCH 1/7] do a bit of housekeeping --- checks/cli.go | 2 +- checks/jq.go | 36 ++++++++-------------- checks/jq_test.go | 12 +++----- checks/local.go | 73 +++++++++++++------------------------------- checks/local_test.go | 27 +++++++++++++++- 5 files changed, 65 insertions(+), 85 deletions(-) diff --git a/checks/cli.go b/checks/cli.go index b9fbf50..d933a6a 100644 --- a/checks/cli.go +++ b/checks/cli.go @@ -178,7 +178,7 @@ func prettyPrintCLICommand(test api.CLICommandTest, variables map[string]string) } if test.StdoutJq != nil { - descriptions = append(descriptions, prettyPrintStdoutJqTest(*test.StdoutJq, variables)) + descriptions = append(descriptions, prettyPrintStdoutJqTest(*test.StdoutJq)) } return strings.Join(descriptions, "\n") diff --git a/checks/jq.go b/checks/jq.go index d9ef4e4..d844f8a 100644 --- a/checks/jq.go +++ b/checks/jq.go @@ -2,33 +2,32 @@ package checks import ( "bytes" - "encoding/json" "errors" "fmt" "io" "strings" api "github.com/bootdotdev/bootdev/client" + "github.com/goccy/go-json" "github.com/itchyny/gojq" "github.com/tailscale/hujson" ) -func prettyPrintStdoutJqTest(test api.StdoutJqTest, variables map[string]string) string { - queryText := test.Query +func prettyPrintStdoutJqTest(test api.StdoutJqTest) string { var str strings.Builder - fmt.Fprintf(&str, "Expect jq query '%s' to yield values satisfying:", queryText) + fmt.Fprintf(&str, "Expect jq query '%s' to yield values satisfying:", test.Query) if len(test.ExpectedResults) == 0 { str.WriteString("\n - [no expected results provided]") return str.String() } for _, expected := range test.ExpectedResults { - value := formatJqExpectedValue(expected, variables) + value := formatJqExpectedValue(expected) fmt.Fprintf(&str, "\n - %s %s %s", expected.Type, expected.Operator, value) } return str.String() } -func formatJqExpectedValue(expected api.JqExpectedResult, variables map[string]string) string { +func formatJqExpectedValue(expected api.JqExpectedResult) string { value := expected.Value encoded, err := json.Marshal(value) if err != nil { @@ -43,31 +42,27 @@ func collectStdoutJqOutputs(cmd api.CLIStepCLICommand, result api.CLICommandResu if test.StdoutJq == nil { continue } - outputs = append(outputs, runStdoutJqQuery(result.Stdout, *test.StdoutJq, result.Variables)) + outputs = append(outputs, runStdoutJqQuery(result.Stdout, *test.StdoutJq)) } return outputs } -func runStdoutJqQuery(stdout string, test api.StdoutJqTest, variables map[string]string) api.CLICommandJqOutput { - queryText := test.Query +func runStdoutJqQuery(stdout string, test api.StdoutJqTest) api.CLICommandJqOutput { input, err := parseJqInput(stdout, test.InputMode) if err != nil { - return api.CLICommandJqOutput{Query: queryText, Error: err.Error()} + return api.CLICommandJqOutput{Query: test.Query, Error: err.Error()} } - results, err := executeJqQuery(queryText, input) + results, err := executeJqQuery(test.Query, input) if err != nil { - return api.CLICommandJqOutput{Query: queryText, Error: err.Error()} + return api.CLICommandJqOutput{Query: test.Query, Error: err.Error()} } - return api.CLICommandJqOutput{Query: queryText, Results: formatJqResults(results)} + return api.CLICommandJqOutput{Query: test.Query, Results: formatJqResults(results)} } func parseJqInput(stdout string, inputMode string) (any, error) { mode := strings.ToLower(strings.TrimSpace(inputMode)) - if mode != "jsonc" && mode != "jsonl" { - mode = "jsonc" - } var inputReader io.Reader - if mode == "jsonc" { + if mode != "jsonl" { // HuJSON requires a newline to terminate a final line comment. standardJSON, err := hujson.Standardize([]byte(stdout + "\n")) if err != nil { @@ -100,13 +95,6 @@ func parseJqInput(stdout string, inputMode string) (any, error) { if err := decoder.Decode(&value); err != nil { return nil, err } - if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { - if err == nil { - return nil, errors.New("expected a single JSON value") - } - return nil, err - } - return value, nil } diff --git a/checks/jq_test.go b/checks/jq_test.go index 2fedd06..70850ee 100644 --- a/checks/jq_test.go +++ b/checks/jq_test.go @@ -12,24 +12,22 @@ func TestRunStdoutJqQuery(t *testing.T) { name string stdout string test api.StdoutJqTest - variables map[string]string want api.CLICommandJqOutput wantError bool }{ { name: "queries JSON with comments using a literal query", stdout: `{ - // Users to query - "users": [/* users */ {"name":"Lane"},{"name":"Theo",},], - }`, + // Users to query + "users": [/* users */ {"name":"Lane"},{"name":"${name}",},], + }`, test: api.StdoutJqTest{ InputMode: "json", Query: `.users[] | select(.name == "${name}") | .name`, }, - variables: map[string]string{"name": "Theo"}, want: api.CLICommandJqOutput{ Query: `.users[] | select(.name == "${name}") | .name`, - Results: nil, + Results: []string{`"${name}"`}, }, }, { @@ -90,7 +88,7 @@ func TestRunStdoutJqQuery(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := runStdoutJqQuery(tt.stdout, tt.test, tt.variables) + got := runStdoutJqQuery(tt.stdout, tt.test) if tt.wantError { if got.Query != tt.want.Query { t.Fatalf("Query = %q, want %q", got.Query, tt.want.Query) diff --git a/checks/local.go b/checks/local.go index e76e2f9..dd33c6e 100644 --- a/checks/local.go +++ b/checks/local.go @@ -1,7 +1,6 @@ package checks import ( - "encoding/json" "errors" "fmt" "math" @@ -11,6 +10,7 @@ import ( "strings" api "github.com/bootdotdev/bootdev/client" + "github.com/goccy/go-json" ) // Local grading mirrors the backend; success is represented by nil. @@ -61,7 +61,10 @@ func EvaluateCLIResults(cliData api.CLIData, results []api.CLIStepResult) *api.S func evaluateCLICommandTests(stepIndex int, expect api.CLIStepCLICommand, actual api.CLICommandResult) *api.StructuredErrCLI { if err := validateCommandAssertions(expect); err != nil { - return &api.StructuredErrCLI{ErrorMessage: err.Error(), FailedStepIndex: stepIndex, FailedTestIndex: -1} + return localFailure(stepIndex, -1, err.Error()) + } + if actual.Err != "" { + return localFailure(stepIndex, -1, actual.Err) } if actual.ExitCode < 0 { return localFailure(stepIndex, -1, "failed to start command") @@ -70,7 +73,7 @@ func evaluateCLICommandTests(stepIndex int, expect api.CLIStepCLICommand, actual for i, expectedTest := range expect.Tests { if expectedTest.ExitCode != nil { if *expectedTest.ExitCode != actual.ExitCode { - return localFailure(stepIndex, i, fmt.Sprintf("expected status code %v, got %v", *expectedTest.ExitCode, actual.ExitCode)) + return localFailure(stepIndex, i, fmt.Sprintf("expected exit code %v, got %v", *expectedTest.ExitCode, actual.ExitCode)) } } if expectedTest.StdoutJq != nil { @@ -127,7 +130,7 @@ func evaluateCLICommandTests(stepIndex int, expect api.CLIStepCLICommand, actual func evaluateHTTPRequestTests(stepIndex int, expect api.CLIStepHTTPRequest, actual api.HTTPRequestResult) *api.StructuredErrCLI { if err := validateHTTPAssertions(expect); err != nil { - return &api.StructuredErrCLI{ErrorMessage: err.Error(), FailedStepIndex: stepIndex, FailedTestIndex: -1} + return localFailure(stepIndex, -1, err.Error()) } if actual.Err != "" { return localFailure(stepIndex, -1, fmt.Sprintf("fetch error: %v", actual.Err)) @@ -183,7 +186,7 @@ func evaluateHTTPRequestTests(stepIndex int, expect api.CLIStepHTTPRequest, actu if expectedTest.JSONValue != nil { err := jsonValOp(*expectedTest.JSONValue, actual.BodyString, actual.Variables) if err != nil { - return localFailure(stepIndex, i, fmt.Sprintf("%v", err)) + return localFailure(stepIndex, i, err.Error()) } } } @@ -227,7 +230,7 @@ func capturedVariableMatches(vars map[string]string, name, expectedValue string) func responseVariableValue(expectedVar api.HTTPRequestResponseVariable, body string) (string, bool) { if expectedVar.Path != "" { val, err := valFromJqPath(expectedVar.Path, body) - if err != nil || val == nil { + if err != nil { return "", false } return fmt.Sprintf("%v", val), true @@ -318,28 +321,24 @@ func jsonValOp(test api.HTTPRequestTestJSONValue, jsn string, variables map[stri if !ok { return errors.New("expected string value") } - if test.Operator == api.OpEquals { - interpolatedStr := InterpolateVariables(*test.StringValue, variables) + interpolatedStr := InterpolateVariables(*test.StringValue, variables) + switch test.Operator { + case api.OpEquals: if vStr != interpolatedStr { return errors.New("string value not equal") } - return nil - } - if test.Operator == api.OpContains { - interpolatedStr := InterpolateVariables(*test.StringValue, variables) + case api.OpContains: if !strings.Contains(vStr, interpolatedStr) { return fmt.Errorf("%s does not contain %s", vStr, interpolatedStr) } - return nil - } - if test.Operator == api.OpNotContains { - interpolatedStr := InterpolateVariables(*test.StringValue, variables) + case api.OpNotContains: if strings.Contains(vStr, interpolatedStr) { return fmt.Errorf("%s contains %s", vStr, interpolatedStr) } - return nil + default: + return errors.New("operator not supported") } - return errors.New("operator not supported") + return nil } return errors.New("no test value provided") @@ -353,14 +352,11 @@ func jqResultMatches(actualResult any, expectedResult api.JqExpectedResult) bool if !expectedOk || !actualOk { return false } - return compareBool(actual, expected, expectedResult.Operator) + return expectedResult.Operator == "==" && actual == expected case api.JqTypeString: - expected, expectedOk := coerceString(expectedResult.Value) - actual, actualOk := coerceString(actualResult) - if !expectedOk || !actualOk { - return false - } - return compareString(actual, expected, expectedResult.Operator) + expected, expectedOk := expectedResult.Value.(string) + actual, actualOk := actualResult.(string) + return expectedOk && actualOk && expectedResult.Operator == "==" && actual == expected case api.JqTypeInt: expected, expectedOk := coerceInt(expectedResult.Value) actual, actualOk := coerceInt(actualResult) @@ -388,15 +384,6 @@ func coerceBool(value any) (bool, bool) { } } -func coerceString(value any) (string, bool) { - switch typed := value.(type) { - case string: - return typed, true - default: - return "", false - } -} - func coerceInt(value any) (int, bool) { switch typed := value.(type) { case int: @@ -435,24 +422,6 @@ func coerceInt(value any) (int, bool) { } } -func compareBool(actual bool, expected bool, operator api.JqOperator) bool { - switch operator { - case "==": - return actual == expected - default: - return false - } -} - -func compareString(actual string, expected string, operator api.JqOperator) bool { - switch operator { - case "==": - return actual == expected - default: - return false - } -} - func compareInt(actual int, expected int, operator api.JqOperator) bool { switch operator { case "==": diff --git a/checks/local_test.go b/checks/local_test.go index 074b80f..41da09f 100644 --- a/checks/local_test.go +++ b/checks/local_test.go @@ -1,12 +1,12 @@ package checks import ( - "encoding/json" "math" "strconv" "testing" api "github.com/bootdotdev/bootdev/client" + "github.com/goccy/go-json" ) func TestLocalSubmissionEventPassesCLIAndHTTPResults(t *testing.T) { @@ -77,6 +77,31 @@ func TestLocalSubmissionEventReportsFirstFailure(t *testing.T) { } } +func TestLocalSubmissionEventRejectsCommandCaptureError(t *testing.T) { + command := api.CLIStepCLICommand{ + Command: "echo hello", + StdoutVariables: []api.CLICommandStdoutVariable{{ + Name: "value", Regex: "(", + }}, + Tests: []api.CLICommandTest{{ExitCode: intPtr(0)}}, + } + result := runCLICommand(command, map[string]string{}, defaultShell()) + if result.ExitCode != 0 || result.Err == "" { + t.Fatalf("expected successful command with capture error, got %#v", result) + } + event := LocalSubmissionEvent( + api.CLIData{Steps: []api.CLIStep{{CLICommand: &command}}}, + []api.CLIStepResult{{CLICommandResult: &result}}, + ) + if event.ResultSlug != api.VerificationResultSlugFailure || event.StructuredErrCLI == nil { + t.Fatalf("expected capture error to fail grading, got %#v", event) + } + failure := event.StructuredErrCLI + if failure.ErrorMessage != result.Err || failure.FailedStepIndex != 0 || failure.FailedTestIndex != -1 { + t.Fatalf("unexpected failure: %#v", failure) + } +} + func TestEvaluateStdoutJqNumericComparisons(t *testing.T) { for _, tt := range []struct { operator api.JqOperator From a343e7b520874dbae1691a6c6100734ccf1ddde9 Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:03:41 -0400 Subject: [PATCH 2/7] consolidate or remove low-value test code --- checks/cli_test.go | 28 +------ checks/http_test.go | 56 +------------- checks/jq_test.go | 163 ++++++++------------------------------- checks/local_test.go | 111 +++++++++----------------- checks/tmdl_test.go | 49 +++--------- client/lessons_test.go | 15 ---- cmd/login_test.go | 60 ++++++-------- render/variables_test.go | 43 +---------- render/view_test.go | 90 +-------------------- 9 files changed, 116 insertions(+), 499 deletions(-) diff --git a/checks/cli_test.go b/checks/cli_test.go index 8073b29..9c1d043 100644 --- a/checks/cli_test.go +++ b/checks/cli_test.go @@ -42,27 +42,6 @@ func TestRunCLICommandCapsOutput(t *testing.T) { } } -func TestRunCLICommandCapturesStdoutVariables(t *testing.T) { - variables := map[string]string{} - result := runCLICommand(api.CLIStepCLICommand{ - Command: `go env GOOS`, - StdoutVariables: []api.CLICommandStdoutVariable{{ - Name: "goos", - Regex: `([a-z0-9]+)`, - }}, - }, variables, defaultShell()) - - if result.Err != "" { - t.Fatalf("unexpected command error: %s", result.Err) - } - if result.Variables["goos"] != runtime.GOOS { - t.Fatalf("captured goos = %q, want %q", result.Variables["goos"], runtime.GOOS) - } - if variables["goos"] != runtime.GOOS { - t.Fatalf("shared goos = %q, want %q", variables["goos"], runtime.GOOS) - } -} - func TestRunCLICommandKeepsStderrSeparateFromStdoutChecks(t *testing.T) { command := `printf 'stdout-value\n'; printf 'stderr-value\n' >&2` if runtime.GOOS == "windows" { @@ -89,9 +68,6 @@ func TestRunCLICommandKeepsStderrSeparateFromStdoutChecks(t *testing.T) { if result.Stderr != "stderr-value" { t.Fatalf("stderr = %q, want stderr-value", result.Stderr) } - if strings.Contains(result.Stdout, "stderr-value") { - t.Fatalf("stdout unexpectedly contains stderr: %q", result.Stdout) - } if _, ok := variables["stderr_value"]; ok { t.Fatalf("stderr unexpectedly populated a stdout variable") } @@ -114,6 +90,10 @@ func TestRunCLICommandInterpolatesCapturedStdoutVariables(t *testing.T) { t.Fatalf("unexpected first command error: %s", first.Err) } + if first.Variables["goenv"] != "GOOS" { + t.Fatalf("captured variable = %q, want GOOS", first.Variables["goenv"]) + } + second := runCLICommand(api.CLIStepCLICommand{ Command: `go env ${goenv}`, }, variables, defaultShell()) diff --git a/checks/http_test.go b/checks/http_test.go index 02436ec..ec2d3a4 100644 --- a/checks/http_test.go +++ b/checks/http_test.go @@ -5,7 +5,6 @@ import ( "io" "net/http" "net/http/httptest" - "slices" "strings" "testing" @@ -18,22 +17,6 @@ func (f httpRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } -type endlessReadCloser struct { - bytesRead int -} - -func (r *endlessReadCloser) Read(p []byte) (int, error) { - for i := range p { - p[i] = 'a' - } - r.bytesRead += len(p) - return len(p), nil -} - -func (r *endlessReadCloser) Close() error { - return nil -} - func TestInterpolateVariables(t *testing.T) { got := InterpolateVariables( "${baseURL}/users/${id}?missing=${missing}", @@ -45,14 +28,6 @@ func TestInterpolateVariables(t *testing.T) { } } -func TestInterpolationNames(t *testing.T) { - got := InterpolationNames("${baseURL}/users/${id}/${id}") - want := []string{"baseURL", "id", "id"} - if !slices.Equal(got, want) { - t.Fatalf("InterpolationNames() = %#v, want %#v", got, want) - } -} - func TestRunHTTPRequestInterpolatesRequestAndCapturesResponseVariables(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -217,13 +192,13 @@ func TestTruncateAndStringifyBodyCapsBinaryBody(t *testing.T) { } func TestRunHTTPRequestCapsResponseBodyRead(t *testing.T) { - body := &endlessReadCloser{} + body := strings.NewReader(strings.Repeat("a", maxHTTPResponseBodyBytes+100)) client := &http.Client{ Transport: httpRoundTripFunc(func(r *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), - Body: body, + Body: io.NopCloser(body), Request: r, }, nil }), @@ -239,8 +214,8 @@ func TestRunHTTPRequestCapsResponseBodyRead(t *testing.T) { if result.Err != "" { t.Fatalf("runHTTPRequest() error = %q", result.Err) } - if body.bytesRead != maxHTTPResponseBodyBytes+1 { - t.Fatalf("response bytes read = %d, want %d", body.bytesRead, maxHTTPResponseBodyBytes+1) + if read := body.Size() - int64(body.Len()); read != maxHTTPResponseBodyBytes+1 { + t.Fatalf("response bytes read = %d, want %d", read, maxHTTPResponseBodyBytes+1) } if len(result.BodyString) != maxHTTPResponseBodyBytes { t.Fatalf("stored response body length = %d, want %d", len(result.BodyString), maxHTTPResponseBodyBytes) @@ -274,11 +249,6 @@ func TestRunHTTPRequestCapturesResponseHeaderVariableAndDoesNotFollowRedirect(t Method: http.MethodPost, FullURL: api.BaseURLPlaceholder + "/login", FollowRedirects: &followRedirects, - BodyForm: map[string]string{ - "email": "pacifica@example.com", - "password": "password123", - "returnTo": "/account", - }, }, } @@ -289,9 +259,6 @@ func TestRunHTTPRequestCapturesResponseHeaderVariableAndDoesNotFollowRedirect(t if result.StatusCode != http.StatusFound { t.Fatalf("StatusCode = %d, want %d", result.StatusCode, http.StatusFound) } - if result.ResponseHeaders["Set-Cookie"] == "" { - t.Fatalf("expected Set-Cookie response header") - } if result.Variables["sessionID"] != "abc123" { t.Fatalf("captured sessionID = %q, want abc123", result.Variables["sessionID"]) } @@ -343,21 +310,6 @@ func TestParseVariablesCapturesBodyRegex(t *testing.T) { } } -func TestParseVariablesRequiresCaptureSource(t *testing.T) { - variables := map[string]string{} - err := parseVariables( - []byte(`{"token":"abc123"}`), - []api.HTTPRequestResponseVariable{{Name: "token"}}, - variables, - ) - if err == nil { - t.Fatal("expected parseVariables error") - } - if err.Error() != "invalid response variable configuration" { - t.Fatalf("error = %q, want invalid response variable configuration", err.Error()) - } -} - func TestParseHeaderVariablesLeavesMissingValuesUnset(t *testing.T) { variables := map[string]string{} err := parseHeaderVariables( diff --git a/checks/jq_test.go b/checks/jq_test.go index 70850ee..2177282 100644 --- a/checks/jq_test.go +++ b/checks/jq_test.go @@ -1,166 +1,67 @@ package checks import ( - "reflect" + "slices" "testing" api "github.com/bootdotdev/bootdev/client" ) func TestRunStdoutJqQuery(t *testing.T) { - tests := []struct { - name string - stdout string - test api.StdoutJqTest - want api.CLICommandJqOutput - wantError bool + for _, tt := range []struct { + name, mode, stdout, query string + want []string + wantError bool }{ { - name: "queries JSON with comments using a literal query", + name: "JSON comments and literal query", mode: "json", stdout: `{ // Users to query "users": [/* users */ {"name":"Lane"},{"name":"${name}",},], }`, - test: api.StdoutJqTest{ - InputMode: "json", - Query: `.users[] | select(.name == "${name}") | .name`, - }, - want: api.CLICommandJqOutput{ - Query: `.users[] | select(.name == "${name}") | .name`, - Results: []string{`"${name}"`}, - }, + query: `.users[] | select(.name == "${name}") | .name`, + want: []string{`"${name}"`}, }, { - name: "default mode accepts comments and trailing commas", - stdout: `{"name": /* user */ "Boots",} // final comment without newline`, - test: api.StdoutJqTest{Query: `.name`}, - want: api.CLICommandJqOutput{ - Query: `.name`, - Results: []string{`"Boots"`}, - }, + name: "default mode and final comment without newline", + stdout: `{"name": "Boots",} // final comment`, query: `.name`, + want: []string{`"Boots"`}, }, { - name: "preserves large integers", - stdout: `{"id":9007199254740993,}`, - test: api.StdoutJqTest{InputMode: "json", Query: `.id`}, - want: api.CLICommandJqOutput{ - Query: `.id`, - Results: []string{`9007199254740993`}, - }, + name: "preserves large integers", mode: "json", + stdout: `{"id":9007199254740993,}`, query: `.id`, + want: []string{`9007199254740993`}, }, { - name: "queries jsonl as array", - stdout: "{\"id\":1}\n{\"id\":2}\n", - test: api.StdoutJqTest{ - InputMode: "jsonl", - Query: `.[].id`, - }, - want: api.CLICommandJqOutput{ - Query: `.[].id`, - Results: []string{`1`, `2`}, - }, + name: "JSONL as array", mode: "jsonl", + stdout: "{\"id\":1}\n{\"id\":2}\n", query: `.[].id`, + want: []string{`1`, `2`}, }, { - name: "returns parse error", - stdout: `{"name":"Boots"} /* unterminated`, - test: api.StdoutJqTest{ - InputMode: "json", - Query: `.name`, - }, - want: api.CLICommandJqOutput{ - Query: `.name`, - }, - wantError: true, + name: "unterminated comment", stdout: `{"name":"Boots"} /* unterminated`, + query: `.name`, wantError: true, }, { - name: "returns jq error", - stdout: `{"name":"Kaladin"}`, - test: api.StdoutJqTest{ - InputMode: "json", - Query: `.name[`, - }, - want: api.CLICommandJqOutput{ - Query: `.name[`, - }, - wantError: true, + name: "invalid query", stdout: `{"name":"Boots"}`, + query: `.name[`, wantError: true, }, - } - - for _, tt := range tests { + { + name: "multiple JSON values", mode: "json", stdout: `{"id":1} {"id":2}`, + query: `.id`, wantError: true, + }, + } { t.Run(tt.name, func(t *testing.T) { - got := runStdoutJqQuery(tt.stdout, tt.test) - if tt.wantError { - if got.Query != tt.want.Query { - t.Fatalf("Query = %q, want %q", got.Query, tt.want.Query) - } - if got.Error == "" { - t.Fatal("expected an error") - } - if len(got.Results) != 0 { - t.Fatalf("expected no results on error, got %v", got.Results) - } - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("runStdoutJqQuery() = %#v, want %#v", got, tt.want) + got := runStdoutJqQuery(tt.stdout, api.StdoutJqTest{InputMode: tt.mode, Query: tt.query}) + if got.Query != tt.query || (got.Error != "") != tt.wantError || !slices.Equal(got.Results, tt.want) { + t.Fatalf("got %#v; want query %q, results %v, error %t", got, tt.query, tt.want, tt.wantError) } }) } } -func TestParseJqInputRejectsMultipleJSONValuesInJSONMode(t *testing.T) { - _, err := parseJqInput("{\"id\":1}\n{\"id\":2}\n", "json") +func TestValFromJqPathRejectsMultipleValues(t *testing.T) { + _, err := valFromJqPath(`.items[].id`, `{"items":[{"id":1},{"id":2}]}`) if err == nil { - t.Fatal("expected error for multiple JSON values in json mode") - } -} - -func TestValFromJqPath(t *testing.T) { - tests := []struct { - name string - path string - jsn string - want any - wantErr string - }{ - { - name: "returns one value", - path: `.token`, - jsn: `{"token":"abc123"}`, - want: "abc123", - }, - { - name: "errors on missing value", - path: `.missing`, - jsn: `{"token":"abc123"}`, - wantErr: "value not found", - }, - { - name: "errors on multiple values", - path: `.items[].id`, - jsn: `{"items":[{"id":1},{"id":2}]}`, - wantErr: "invalid number of values found", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := valFromJqPath(tt.path, tt.jsn) - if tt.wantErr != "" { - if err == nil { - t.Fatalf("expected error %q", tt.wantErr) - } - if err.Error() != tt.wantErr { - t.Fatalf("expected error %q, got %q", tt.wantErr, err.Error()) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !reflect.DeepEqual(got, tt.want) { - t.Fatalf("valFromJqPath() = %#v, want %#v", got, tt.want) - } - }) + t.Fatal("expected error for multiple values") } } diff --git a/checks/local_test.go b/checks/local_test.go index 41da09f..f0b1537 100644 --- a/checks/local_test.go +++ b/checks/local_test.go @@ -9,44 +9,25 @@ import ( "github.com/goccy/go-json" ) -func TestLocalSubmissionEventPassesCLIAndHTTPResults(t *testing.T) { - cliData := api.CLIData{Steps: []api.CLIStep{ - {CLICommand: &api.CLIStepCLICommand{Tests: []api.CLICommandTest{ - {ExitCode: intPtr(0)}, - {StdoutContainsAll: []string{"hello ${name}"}}, - }}}, - {HTTPRequest: &api.CLIStepHTTPRequest{Tests: []api.HTTPRequestTest{ - {StatusCode: intPtr(200)}, - {HeadersEqual: &api.HTTPRequestTestHeader{Key: "Set-Cookie", Value: "session_id=abc123; Path=/"}}, - {HeadersContain: &api.HTTPRequestTestHeader{Key: "Set-Cookie", Value: "session_id="}}, - {JSONValue: &api.HTTPRequestTestJSONValue{ - Path: ".app", - Operator: api.OpEquals, - StringValue: stringPtr("bearly-secure"), - }}, - }}}, +func TestLocalSubmissionEventInterpolatesExpectedValues(t *testing.T) { + data := api.CLIData{Steps: []api.CLIStep{ + {CLICommand: &api.CLIStepCLICommand{Tests: []api.CLICommandTest{{ + StdoutContainsAll: []string{"hello ${name}"}, + }}}}, + {HTTPRequest: &api.CLIStepHTTPRequest{Tests: []api.HTTPRequestTest{{ + JSONValue: &api.HTTPRequestTestJSONValue{ + Path: ".name", Operator: api.OpEquals, StringValue: stringPtr("${name}"), + }, + }}}}, }} - + variables := map[string]string{"name": "Boots"} results := []api.CLIStepResult{ - {CLICommandResult: &api.CLICommandResult{ - ExitCode: 0, - Stdout: "hello Boots", - Variables: map[string]string{"name": "Boots"}, - }}, - {HTTPRequestResult: &api.HTTPRequestResult{ - StatusCode: 200, - ResponseHeaders: map[string]string{"Set-Cookie": "session_id=abc123; Path=/"}, - BodyString: `{"app":"bearly-secure"}`, - Variables: map[string]string{}, - }}, + {CLICommandResult: &api.CLICommandResult{Stdout: "hello Boots", Variables: variables}}, + {HTTPRequestResult: &api.HTTPRequestResult{BodyString: `{"name":"Boots"}`, Variables: variables}}, } - - event := LocalSubmissionEvent(cliData, results) + event := LocalSubmissionEvent(data, results) if event.ResultSlug != api.VerificationResultSlugSuccess { - t.Fatalf("ResultSlug = %q, want success; failure = %#v", event.ResultSlug, event.StructuredErrCLI) - } - if event.StructuredErrCLI != nil { - t.Fatalf("unexpected failure: %#v", event.StructuredErrCLI) + t.Fatalf("expected interpolated assertions to pass, got %#v", event.StructuredErrCLI) } } @@ -55,6 +36,7 @@ func TestLocalSubmissionEventReportsFirstFailure(t *testing.T) { {CLICommand: &api.CLIStepCLICommand{Tests: []api.CLICommandTest{ {ExitCode: intPtr(0)}, {StdoutContainsAll: []string{"expected"}}, + {StdoutContainsAll: []string{"also missing"}}, }}}, }} results := []api.CLIStepResult{ @@ -134,60 +116,40 @@ func TestEvaluateStdoutJqNumericComparisons(t *testing.T) { } func TestEvaluateHTTPRequestTestsHeaderAndTrailerEquality(t *testing.T) { - tests := []struct { + header := &api.HTTPRequestTestHeader{Key: "X-Request-ID", Value: "abc123"} + for _, tt := range []struct { name string test api.HTTPRequestTest result api.HTTPRequestResult wantFailure bool }{ { - name: "header name is case insensitive", - test: api.HTTPRequestTest{HeadersEqual: &api.HTTPRequestTestHeader{ - Key: "X-Request-ID", - Value: "abc123", - }}, - result: api.HTTPRequestResult{ - ResponseHeaders: map[string]string{"x-request-id": "abc123"}, - }, + "header name ignores case", + api.HTTPRequestTest{HeadersEqual: header}, + api.HTTPRequestResult{ResponseHeaders: map[string]string{"x-request-id": "abc123"}}, + false, }, { - name: "header value is case sensitive", - test: api.HTTPRequestTest{HeadersEqual: &api.HTTPRequestTestHeader{ - Key: "X-Request-ID", - Value: "abc123", - }}, - result: api.HTTPRequestResult{ - ResponseHeaders: map[string]string{"X-Request-ID": "ABC123"}, - }, - wantFailure: true, + "header value preserves case", + api.HTTPRequestTest{HeadersEqual: header}, + api.HTTPRequestResult{ResponseHeaders: map[string]string{"X-Request-ID": "ABC123"}}, + true, }, { - name: "trailer name is case insensitive", - test: api.HTTPRequestTest{TrailersEqual: &api.HTTPRequestTestHeader{ - Key: "X-Checksum", - Value: "sha256:abc", - }}, - result: api.HTTPRequestResult{ - ResponseTrailers: map[string]string{"x-checksum": "sha256:abc"}, - }, + "trailer name ignores case", + api.HTTPRequestTest{TrailersEqual: header}, + api.HTTPRequestResult{ResponseTrailers: map[string]string{"x-request-id": "abc123"}}, + false, }, { - name: "trailer value is case sensitive", - test: api.HTTPRequestTest{TrailersEqual: &api.HTTPRequestTestHeader{ - Key: "X-Checksum", - Value: "sha256:abc", - }}, - result: api.HTTPRequestResult{ - ResponseTrailers: map[string]string{"X-Checksum": "SHA256:ABC"}, - }, - wantFailure: true, + "trailer value preserves case", + api.HTTPRequestTest{TrailersEqual: header}, + api.HTTPRequestResult{ResponseTrailers: map[string]string{"X-Request-ID": "ABC123"}}, + true, }, - } - - for _, tt := range tests { + } { t.Run(tt.name, func(t *testing.T) { - request := api.CLIStepHTTPRequest{Tests: []api.HTTPRequestTest{tt.test}} - failure := evaluateHTTPRequestTests(0, request, tt.result) + failure := evaluateHTTPRequestTests(0, api.CLIStepHTTPRequest{Tests: []api.HTTPRequestTest{tt.test}}, tt.result) if (failure != nil) != tt.wantFailure { t.Fatalf("failure = %#v, wantFailure = %t", failure, tt.wantFailure) } @@ -289,7 +251,6 @@ func TestEvaluateStdoutJqMatchesAnyResult(t *testing.T) { {"reuse an actual result", "[1]", []int{1, 1}, true}, {"missing expected result", "[1, 3]", []int{1, 2}, false}, {"empty results", "[]", []int{1}, false}, - {"empty results without expectations", "[]", nil, false}, {"nonempty results without expectations", "[1]", nil, false}, } { t.Run(tt.name, func(t *testing.T) { diff --git a/checks/tmdl_test.go b/checks/tmdl_test.go index 08a9499..4e265f0 100644 --- a/checks/tmdl_test.go +++ b/checks/tmdl_test.go @@ -4,50 +4,19 @@ import "testing" func TestExtractTmdlBlock(t *testing.T) { input := "root\n child one\n grandchild\n\n child two\nnext root" - - tests := []struct { - name string - query string - want string + for _, tt := range []struct { + name, input, query, want string }{ - { - name: "blank query returns input", - query: " ", - want: input, - }, - { - name: "missing query returns empty string", - query: "missing", - want: "", - }, - { - name: "extracts matching item block", - query: "child one", - want: " child one\n grandchild", - }, - { - name: "blank lines do not terminate block", - query: "root", - want: "root\n child one\n grandchild\n\n child two", - }, - } - - for _, tt := range tests { + {"blank query", input, " ", input}, + {"missing query", input, "missing", ""}, + {"nested block", input, "child one", " child one\n grandchild"}, + {"blank lines inside block", input, "root", "root\n child one\n grandchild\n\n child two"}, + {"tabs", "item\n\tchild\n\t\tgrandchild\n\tsibling\nnext", "child", "\tchild\n\t\tgrandchild"}, + } { t.Run(tt.name, func(t *testing.T) { - got := ExtractTmdlBlock(input, tt.query) - if got != tt.want { + if got := ExtractTmdlBlock(tt.input, tt.query); got != tt.want { t.Fatalf("ExtractTmdlBlock() = %q, want %q", got, tt.want) } }) } } - -func TestExtractTmdlBlockHandlesTabIndentation(t *testing.T) { - input := "item\n\tchild\n\t\tgrandchild\n\tsibling\nnext" - - got := ExtractTmdlBlock(input, "child") - want := "\tchild\n\t\tgrandchild" - if got != want { - t.Fatalf("ExtractTmdlBlock() = %q, want %q", got, want) - } -} diff --git a/client/lessons_test.go b/client/lessons_test.go index 7b6fd12..7a68782 100644 --- a/client/lessons_test.go +++ b/client/lessons_test.go @@ -42,18 +42,3 @@ func TestHTTPRequestResultSerializesFetchError(t *testing.T) { t.Fatalf("FetchErr = %v, want %q; payload: %s", submitted.FetchErr, fetchErr, payload) } } - -func TestHTTPRequestResultOmitsEmptyFetchError(t *testing.T) { - payload, err := json.Marshal(HTTPRequestResult{}) - if err != nil { - t.Fatalf("marshal HTTP request result: %v", err) - } - - var submitted map[string]any - if err := json.Unmarshal(payload, &submitted); err != nil { - t.Fatalf("unmarshal submission payload: %v", err) - } - if _, ok := submitted["FetchErr"]; ok { - t.Fatalf("submission payload unexpectedly contains an empty FetchErr: %s", payload) - } -} diff --git a/cmd/login_test.go b/cmd/login_test.go index b93b875..add238e 100644 --- a/cmd/login_test.go +++ b/cmd/login_test.go @@ -21,48 +21,36 @@ func TestLoginHTTPHandlerAcceptsConfiguredOrigin(t *testing.T) { if response.Code != http.StatusNoContent { t.Fatalf("status = %d, want %d", response.Code, http.StatusNoContent) } - if code := <-inputChan; code != "valid-code" { - t.Fatalf("login code = %q, want valid-code", code) - } - if origin := response.Header().Get("Access-Control-Allow-Origin"); origin != testFrontendURL { - t.Fatalf("allowed origin = %q, want %q", origin, testFrontendURL) - } -} - -func TestLoginHTTPHandlerRejectsUnexpectedOrigin(t *testing.T) { - inputChan := make(chan string, 1) - handler := newLoginHTTPHandler(inputChan, testFrontendURL) - request := httptest.NewRequest(http.MethodPost, "/submit", strings.NewReader("attacker-code")) - request.Header.Set("Origin", "https://example.com") - response := httptest.NewRecorder() - - handler.ServeHTTP(response, request) - - if response.Code != http.StatusForbidden { - t.Fatalf("status = %d, want %d", response.Code, http.StatusForbidden) - } select { case code := <-inputChan: - t.Fatalf("unexpected login code accepted: %q", code) + if code != "valid-code" { + t.Fatalf("login code = %q, want valid-code", code) + } default: + t.Fatal("login code was not delivered") + } + if origin := response.Header().Get("Access-Control-Allow-Origin"); origin != testFrontendURL { + t.Fatalf("allowed origin = %q, want %q", origin, testFrontendURL) } } -func TestLoginHTTPHandlerRejectsMissingOrigin(t *testing.T) { - inputChan := make(chan string, 1) - handler := newLoginHTTPHandler(inputChan, testFrontendURL) - request := httptest.NewRequest(http.MethodPost, "/submit", strings.NewReader("attacker-code")) - response := httptest.NewRecorder() - - handler.ServeHTTP(response, request) - - if response.Code != http.StatusForbidden { - t.Fatalf("status = %d, want %d", response.Code, http.StatusForbidden) - } - select { - case code := <-inputChan: - t.Fatalf("unexpected login code accepted: %q", code) - default: +func TestLoginHTTPHandlerRejectsUntrustedOrigin(t *testing.T) { + for _, origin := range []string{"https://example.com", ""} { + t.Run(origin, func(t *testing.T) { + inputChan := make(chan string, 1) + request := httptest.NewRequest(http.MethodPost, "/submit", strings.NewReader("attacker-code")) + request.Header.Set("Origin", origin) + response := httptest.NewRecorder() + newLoginHTTPHandler(inputChan, testFrontendURL).ServeHTTP(response, request) + if response.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", response.Code, http.StatusForbidden) + } + select { + case code := <-inputChan: + t.Fatalf("unexpected login code accepted: %q", code) + default: + } + }) } } diff --git a/render/variables_test.go b/render/variables_test.go index b7ed73e..38785ac 100644 --- a/render/variables_test.go +++ b/render/variables_test.go @@ -10,21 +10,17 @@ import ( func TestHTTPVariableSections(t *testing.T) { result := api.HTTPRequestResult{ Variables: map[string]string{ - "authToken": "token-123", - "resetToken": "reset-123", - "shortCode": "abc123", - "sessionID": "session-123", + "authToken": "token-123", + "shortCode": "abc123", + "sessionID": "session-123", }, Request: api.CLIStepHTTPRequest{ ResponseVariables: []api.HTTPRequestResponseVariable{ {Name: "shortCode", Path: ".short_code"}, - {Name: "resetToken", BodyRegex: `/password-reset/([a-z0-9]+)`}, {Name: "missingCode", Path: ".missing_code"}, - {Name: "missingResetToken", BodyRegex: `/missing/([a-z0-9]+)`}, }, ResponseHeaderVariables: []api.HTTPRequestResponseHeaderVariable{ {Name: "sessionID", Header: "Set-Cookie", Regex: "session_id=([^;]+)"}, - {Name: "missingSessionID", Header: "Set-Cookie", Regex: "missing=([^;]+)"}, }, Request: api.HTTPRequest{ FullURL: "${baseURL}/api/links/${shortCode}", @@ -39,13 +35,10 @@ func TestHTTPVariableSections(t *testing.T) { wantContains := []string{ "Variables Saved:", - "resetToken: reset-123 (Response Body pattern)", "sessionID: session-123 (Response Header Set-Cookie pattern)", "shortCode: abc123 (JSON Body .short_code)", "Variables Missing:", "missingCode: [not found] (JSON Body .missing_code)", - "missingResetToken: [not found] (Response Body pattern)", - "missingSessionID: [not found] (Response Header Set-Cookie pattern)", "Variables Available:", "authToken: token-123 (Request Header \"Authorization\")", "shortCode: abc123 (Request URL)", @@ -61,28 +54,6 @@ func TestHTTPVariableSections(t *testing.T) { } } -func TestAvailableVariablesPrintsNotFoundWhenExpectedButUnavailable(t *testing.T) { - result := api.CLICommandResult{ - Variables: map[string]string{}, - Command: api.CLIStepCLICommand{ - Command: "curl ${url}", - }, - } - - available, expectsVariables := availableVariablesForCLIResult(result) - if !expectsVariables { - t.Fatalf("expected CLI command to use variables") - } - got := renderVariableSection("Variables Available", available) - - if !strings.Contains(got, "Variables Available:") { - t.Fatalf("expected Variables Available section in:\n%s", got) - } - if !strings.Contains(got, "url: [not found] (Command)") { - t.Fatalf("expected missing url in:\n%s", got) - } -} - func TestHTTPVariableSectionsDistinguishEmptyFromMissing(t *testing.T) { result := api.HTTPRequestResult{ Variables: map[string]string{"emptyCode": ""}, @@ -107,14 +78,6 @@ func TestHTTPVariableSectionsDistinguishEmptyFromMissing(t *testing.T) { t.Errorf("output missing %q\n%s", expected, got) } } - for _, unexpected := range []string{ - "emptyCode: [not found]", - "missingCode: [empty]", - } { - if strings.Contains(got, unexpected) { - t.Errorf("output unexpectedly contains %q\n%s", unexpected, got) - } - } } func TestCLIAvailableVariables(t *testing.T) { diff --git a/render/view_test.go b/render/view_test.go index 6703506..69a2627 100644 --- a/render/view_test.go +++ b/render/view_test.go @@ -7,7 +7,6 @@ import ( "unicode/utf8" api "github.com/bootdotdev/bootdev/client" - "github.com/bootdotdev/bootdev/messages" ) func TestCompactViewHidesSuccessfulDetailsAndExpandsFailure(t *testing.T) { @@ -92,59 +91,6 @@ func TestCompactViewDoesNotHideStepsForInvalidFailureIndex(t *testing.T) { } } -func TestVerboseViewShowsSuccessfulDetails(t *testing.T) { - passed := true - m := initModel(true, true) - m.finalized = true - m.result = api.VerificationResultSlugSuccess - m.steps = []stepModel{{ - description: "The command prints a greeting", - detail: "Command: echo hello", - passed: &passed, - finished: true, - tests: []testModel{{text: "Expect stdout to contain all of: hello", passed: &passed, finished: true}}, - result: &api.CLIStepResult{CLICommandResult: &api.CLICommandResult{ - Stdout: "hello", - Stderr: "diagnostic message", - }}, - }} - - view := m.View() - for _, expected := range []string{ - "The command prints a greeting", - "Command: echo hello", - "Expect stdout to contain all of: hello", - "Command stdout:", - "hello", - "Command stderr:", - "diagnostic message", - } { - if !strings.Contains(view, expected) { - t.Errorf("view missing %q\n%s", expected, view) - } - } -} - -func TestVerboseViewStaysCompactUntilFinalized(t *testing.T) { - m := initModel(true, true) - m.steps = []stepModel{{ - description: "The command prints a greeting", - detail: "Command: echo hello", - finished: true, - tests: []testModel{{text: "Expect stdout to contain all of: hello", finished: true}}, - }} - - view := m.View() - if !strings.Contains(view, "The command prints a greeting") { - t.Fatalf("view missing compact step description\n%s", view) - } - for _, unexpected := range []string{"Command: echo hello", "Expect stdout to contain all of: hello"} { - if strings.Contains(view, unexpected) { - t.Errorf("view unexpectedly contains %q before finalization\n%s", unexpected, view) - } - } -} - func TestSystemErrorViewDoesNotShowStepsAsPassed(t *testing.T) { m := initModel(true, false) m.finalized = true @@ -157,8 +103,7 @@ func TestSystemErrorViewDoesNotShowStepsAsPassed(t *testing.T) { view := m.View() for _, expected := range []string{ "? The command prints a greeting", - "Unable to verify this lesson due to a system error.", - "Please try again.", + "system error", } { if !strings.Contains(view, expected) { t.Errorf("view missing %q\n%s", expected, view) @@ -171,41 +116,14 @@ func TestSystemErrorViewDoesNotShowStepsAsPassed(t *testing.T) { } } -func TestStartStepFallsBackToTechnicalDescription(t *testing.T) { - m := initModel(true, false) - updated, _ := m.Update(messages.StartStepMsg{CMD: "go test ./..."}) - got := updated.(rootModel).steps[0] - - if got.description != "go test ./..." { - t.Fatalf("description = %q, want command fallback", got.description) - } - if got.detail != "Command: go test ./..." { - t.Fatalf("detail = %q, want technical command", got.detail) - } -} - -func TestCompactStepHonorsSubmitMode(t *testing.T) { - step := stepModel{description: "A completed step", finished: true} - - submit := renderCompactStep(step, "", true) - if !strings.Contains(submit, "? A completed step") { - t.Fatalf("submit output = %q, want unresolved marker", submit) - } - - run := renderCompactStep(step, "", false) - if run != "A completed step\n" { - t.Fatalf("run output = %q, want plain description", run) - } -} - func TestTruncateVisualOutputCapsLinesAndRunes(t *testing.T) { tests := []struct { name string output string }{ - {name: "many lines", output: strings.Repeat("line\n", 100_000)}, - {name: "long ASCII line", output: strings.Repeat("x", 1_000_000)}, - {name: "long Unicode line", output: strings.Repeat("界", 1_000_000)}, + {name: "many lines", output: strings.Repeat("line\n", 40)}, + {name: "long ASCII line", output: strings.Repeat("x", 6000)}, + {name: "long Unicode line", output: strings.Repeat("界", 6000)}, } for _, tt := range tests { From 302fc8b985842c1b8794612cbad27a4744f30240 Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:15:27 -0400 Subject: [PATCH 3/7] consolidate value capture logic --- checks/captures.go | 85 +++++++++++++++++++++++++++++++++++++++++++++ checks/http.go | 68 ------------------------------------ checks/http_test.go | 21 +++++++++++ checks/local.go | 54 +++------------------------- 4 files changed, 110 insertions(+), 118 deletions(-) create mode 100644 checks/captures.go diff --git a/checks/captures.go b/checks/captures.go new file mode 100644 index 0000000..8b5b890 --- /dev/null +++ b/checks/captures.go @@ -0,0 +1,85 @@ +package checks + +import ( + "errors" + "fmt" + "regexp" + + api "github.com/bootdotdev/bootdev/client" +) + +func parseVariables(body []byte, vardefs []api.HTTPRequestResponseVariable, variables map[string]string) error { + bodyString := string(body) + for _, vardef := range vardefs { + value, found, err := responseVariableValue(vardef, bodyString) + if err != nil { + return err + } + if found { + variables[vardef.Name] = value + } + } + return nil +} + +func parseHeaderVariables(headers map[string]string, vardefs []api.HTTPRequestResponseHeaderVariable, variables map[string]string) error { + for _, vardef := range vardefs { + value, found, err := responseHeaderVariableValue(vardef, headers) + if err != nil { + return err + } + if found { + variables[vardef.Name] = value + } + } + return nil +} + +// A found value may be empty. A missing match is distinct from a parsing error. +func responseVariableValue(vardef api.HTTPRequestResponseVariable, body string) (string, bool, error) { + if (vardef.Path == "") == (vardef.BodyRegex == "") { + return "", false, errors.New("invalid response variable configuration") + } + if vardef.BodyRegex != "" { + value, found, err := regexCapture(vardef.BodyRegex, body) + if err != nil { + return "", false, errors.New("invalid response body variable configuration") + } + return value, found, nil + } + values, err := valsFromJqPath(vardef.Path, body) + if err != nil { + return "", false, err + } + if len(values) != 1 || values[0] == nil { + return "", false, nil + } + return fmt.Sprintf("%v", values[0]), true, nil +} + +func responseHeaderVariableValue(vardef api.HTTPRequestResponseHeaderVariable, headers map[string]string) (string, bool, error) { + value, found := findHeaderValue(headers, vardef.Header) + if !found || vardef.Regex == "" { + return value, found, nil + } + value, found, err := regexCapture(vardef.Regex, value) + if err != nil { + return "", false, errors.New("invalid response header variable configuration") + } + return value, found, nil +} + +func regexCapture(pattern, input string) (string, bool, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return "", false, err + } + if re.NumSubexp() != 1 { + return "", false, errors.New("capture regex requires exactly one capture group") + } + matches := re.FindStringSubmatch(input) + if len(matches) != 2 { + return "", false, nil + } + return matches[1], true, nil +} diff --git a/checks/http.go b/checks/http.go index dba42bf..c472f5d 100644 --- a/checks/http.go +++ b/checks/http.go @@ -217,74 +217,6 @@ func truncateAndStringifyBody(body []byte) string { return string(body) } -func parseVariables(body []byte, vardefs []api.HTTPRequestResponseVariable, variables map[string]string) error { - bodyString := string(body) - - for _, vardef := range vardefs { - switch { - case vardef.Path != "" && vardef.BodyRegex != "": - return fmt.Errorf("invalid response variable configuration") - - case vardef.BodyRegex != "": - re, err := regexp.Compile(vardef.BodyRegex) - if err != nil { - return fmt.Errorf("invalid response body variable configuration") - } - if re.NumSubexp() != 1 { - return fmt.Errorf("invalid response body variable configuration") - } - matches := re.FindStringSubmatch(bodyString) - if len(matches) == 2 { - variables[vardef.Name] = matches[1] - } - - case vardef.Path != "": - vals, err := valsFromJqPath(vardef.Path, bodyString) - if err != nil { - return err - } - if len(vals) == 1 && vals[0] != nil { - variables[vardef.Name] = fmt.Sprintf("%v", vals[0]) - } - - default: - return fmt.Errorf("invalid response variable configuration") - } - } - - return nil -} - -func parseHeaderVariables(headers map[string]string, vardefs []api.HTTPRequestResponseHeaderVariable, variables map[string]string) error { - for _, vardef := range vardefs { - headerValue, ok := findHeaderValue(headers, vardef.Header) - if !ok { - continue - } - - value := headerValue - if vardef.Regex != "" { - re, err := regexp.Compile(vardef.Regex) - if err != nil { - return fmt.Errorf("invalid response header variable configuration") - } - if re.NumSubexp() != 1 { - return fmt.Errorf("invalid response header variable configuration") - } - - matches := re.FindStringSubmatch(headerValue) - if len(matches) != 2 { - continue - } - value = matches[1] - } - - variables[vardef.Name] = value - } - - return nil -} - func findHeaderValue(headers map[string]string, key string) (string, bool) { for actualKey, value := range headers { if strings.EqualFold(actualKey, key) { diff --git a/checks/http_test.go b/checks/http_test.go index ec2d3a4..6535960 100644 --- a/checks/http_test.go +++ b/checks/http_test.go @@ -76,6 +76,9 @@ func TestRunHTTPRequestInterpolatesRequestAndCapturesResponseVariables(t *testin } requestStep := api.CLIStepHTTPRequest{ ResponseVariables: []api.HTTPRequestResponseVariable{{Name: "token", Path: ".token"}}, + ResponseHeaderVariables: []api.HTTPRequestResponseHeaderVariable{{ + Name: "requestOK", Header: "x-request-ok", + }}, Request: api.HTTPRequest{ Method: http.MethodPost, FullURL: api.BaseURLPlaceholder + "/users/${id}", @@ -108,6 +111,17 @@ func TestRunHTTPRequestInterpolatesRequestAndCapturesResponseVariables(t *testin if result.Variables["id"] != "42" { t.Fatalf("original variable id = %q, want %q", result.Variables["id"], "42") } + if failure := evaluateHTTPRequestTests(0, requestStep, result); failure != nil { + t.Fatalf("captured response failed grading: %#v", failure) + } + for _, name := range []string{"token", "requestOK"} { + original := result.Variables[name] + result.Variables[name] = "wrong" + if failure := evaluateHTTPRequestTests(0, requestStep, result); failure == nil { + t.Fatalf("grading accepted incorrect capture %q", name) + } + result.Variables[name] = original + } } func TestRunHTTPRequestSafelyInterpolatesNestedJSONStrings(t *testing.T) { @@ -262,6 +276,9 @@ func TestRunHTTPRequestCapturesResponseHeaderVariableAndDoesNotFollowRedirect(t if result.Variables["sessionID"] != "abc123" { t.Fatalf("captured sessionID = %q, want abc123", result.Variables["sessionID"]) } + if failure := evaluateHTTPRequestTests(0, requestStep, result); failure != nil { + t.Fatalf("captured header regex failed grading: %#v", failure) + } } func TestParseVariablesLeavesMissingValuesUnset(t *testing.T) { @@ -272,6 +289,7 @@ func TestParseVariablesLeavesMissingValuesUnset(t *testing.T) { {Name: "token", Path: ".token"}, {Name: "missing", Path: ".missing"}, {Name: "notFound", Path: ".not_found"}, + {Name: "multiple", Path: ".token, .token"}, }, variables, ) @@ -287,6 +305,9 @@ func TestParseVariablesLeavesMissingValuesUnset(t *testing.T) { if _, ok := variables["notFound"]; ok { t.Fatalf("expected missing variable to remain unset") } + if _, ok := variables["multiple"]; ok { + t.Fatal("expected multiple values to remain unset") + } } func TestParseVariablesCapturesBodyRegex(t *testing.T) { diff --git a/checks/local.go b/checks/local.go index dd33c6e..c79b8c7 100644 --- a/checks/local.go +++ b/checks/local.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "math/big" - "regexp" "strconv" "strings" @@ -198,8 +197,8 @@ func evaluateHTTPRequestTests(stepIndex int, expect api.CLIStepHTTPRequest, actu } for _, expectedVar := range expect.ResponseVariables { - expectedValue, ok := responseVariableValue(expectedVar, actual.BodyString) - if !ok { + expectedValue, ok, err := responseVariableValue(expectedVar, actual.BodyString) + if err != nil || !ok { return localFailure(stepIndex, responseVariableTestIndex, fmt.Sprintf("missing value for variable '%s'", expectedVar.Name)) } @@ -209,8 +208,8 @@ func evaluateHTTPRequestTests(stepIndex int, expect api.CLIStepHTTPRequest, actu } for _, expectedVar := range expect.ResponseHeaderVariables { - expectedValue, ok := responseHeaderVariableValue(expectedVar, actual.ResponseHeaders) - if !ok { + expectedValue, ok, err := responseHeaderVariableValue(expectedVar, actual.ResponseHeaders) + if err != nil || !ok { return localFailure(stepIndex, responseHeaderVariableTestIndex, fmt.Sprintf("missing value for variable '%s'", expectedVar.Name)) } @@ -227,51 +226,6 @@ func capturedVariableMatches(vars map[string]string, name, expectedValue string) return ok && actualValue == expectedValue } -func responseVariableValue(expectedVar api.HTTPRequestResponseVariable, body string) (string, bool) { - if expectedVar.Path != "" { - val, err := valFromJqPath(expectedVar.Path, body) - if err != nil { - return "", false - } - return fmt.Sprintf("%v", val), true - } - - re, err := regexp.Compile(expectedVar.BodyRegex) - if err != nil { - return "", false - } - - matches := re.FindStringSubmatch(body) - if len(matches) != 2 { - return "", false - } - - return matches[1], true -} - -func responseHeaderVariableValue(expectedVar api.HTTPRequestResponseHeaderVariable, headers map[string]string) (string, bool) { - headerValue, ok := findHeaderValue(headers, expectedVar.Header) - if !ok { - return "", false - } - - if expectedVar.Regex == "" { - return headerValue, true - } - - re, err := regexp.Compile(expectedVar.Regex) - if err != nil { - return "", false - } - - matches := re.FindStringSubmatch(headerValue) - if len(matches) != 2 { - return "", false - } - - return matches[1], true -} - func jsonValOp(test api.HTTPRequestTestJSONValue, jsn string, variables map[string]string) error { val, err := valFromJqPath(test.Path, jsn) if err != nil { From 015b917f2844910b835643f6203c31d42b703133 Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:17:13 -0400 Subject: [PATCH 4/7] keep jq logic organized --- checks/jq.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++++ checks/local.go | 99 ------------------------------------------------- 2 files changed, 98 insertions(+), 99 deletions(-) diff --git a/checks/jq.go b/checks/jq.go index d844f8a..7053a7d 100644 --- a/checks/jq.go +++ b/checks/jq.go @@ -5,6 +5,9 @@ import ( "errors" "fmt" "io" + "math" + "math/big" + "strconv" "strings" api "github.com/bootdotdev/bootdev/client" @@ -138,6 +141,101 @@ func formatJqResults(results []any) []string { return formatted } +func jqResultMatches(actualResult any, expectedResult api.JqExpectedResult) bool { + switch expectedResult.Type { + case api.JqTypeBool: + expected, expectedOk := coerceBool(expectedResult.Value) + actual, actualOk := coerceBool(actualResult) + if !expectedOk || !actualOk { + return false + } + return expectedResult.Operator == "==" && actual == expected + case api.JqTypeString: + expected, expectedOk := expectedResult.Value.(string) + actual, actualOk := actualResult.(string) + return expectedOk && actualOk && expectedResult.Operator == "==" && actual == expected + case api.JqTypeInt: + expected, expectedOk := coerceInt(expectedResult.Value) + actual, actualOk := coerceInt(actualResult) + if !expectedOk || !actualOk { + return false + } + return compareInt(actual, expected, expectedResult.Operator) + default: + return false + } +} + +func coerceBool(value any) (bool, bool) { + switch typed := value.(type) { + case bool: + return typed, true + case string: + parsed, err := strconv.ParseBool(typed) + if err != nil { + return false, false + } + return parsed, true + default: + return false, false + } +} + +func coerceInt(value any) (int, bool) { + switch typed := value.(type) { + case int: + return typed, true + case int64: + if typed > math.MaxInt || typed < math.MinInt { + return 0, false + } + return int(typed), true + case float64: + if math.IsNaN(typed) || math.IsInf(typed, 0) { + return 0, false + } + if math.Trunc(typed) != typed { + return 0, false + } + // MaxInt rounds up as float64 on 64-bit hosts; use an exclusive upper bound. + if typed >= -float64(math.MinInt) || typed < float64(math.MinInt) { + return 0, false + } + return int(typed), true + case json.Number: + parsed, ok := new(big.Rat).SetString(typed.String()) + if !ok || !parsed.IsInt() || !parsed.Num().IsInt64() { + return 0, false + } + return coerceInt(parsed.Num().Int64()) + case string: + parsed, err := strconv.Atoi(typed) + if err != nil { + return 0, false + } + return parsed, true + default: + return 0, false + } +} + +func compareInt(actual int, expected int, operator api.JqOperator) bool { + switch operator { + case "==": + return actual == expected + case ">": + return actual > expected + case ">=": + return actual >= expected + case "<": + return actual < expected + case "<=": + return actual <= expected + default: + return false + } +} + func valFromJqPath(path string, jsn string) (any, error) { vals, err := valsFromJqPath(path, jsn) if err != nil { diff --git a/checks/local.go b/checks/local.go index c79b8c7..f78851b 100644 --- a/checks/local.go +++ b/checks/local.go @@ -3,13 +3,9 @@ package checks import ( "errors" "fmt" - "math" - "math/big" - "strconv" "strings" api "github.com/bootdotdev/bootdev/client" - "github.com/goccy/go-json" ) // Local grading mirrors the backend; success is represented by nil. @@ -298,101 +294,6 @@ func jsonValOp(test api.HTTPRequestTestJSONValue, jsn string, variables map[stri return errors.New("no test value provided") } -func jqResultMatches(actualResult any, expectedResult api.JqExpectedResult) bool { - switch expectedResult.Type { - case api.JqTypeBool: - expected, expectedOk := coerceBool(expectedResult.Value) - actual, actualOk := coerceBool(actualResult) - if !expectedOk || !actualOk { - return false - } - return expectedResult.Operator == "==" && actual == expected - case api.JqTypeString: - expected, expectedOk := expectedResult.Value.(string) - actual, actualOk := actualResult.(string) - return expectedOk && actualOk && expectedResult.Operator == "==" && actual == expected - case api.JqTypeInt: - expected, expectedOk := coerceInt(expectedResult.Value) - actual, actualOk := coerceInt(actualResult) - if !expectedOk || !actualOk { - return false - } - return compareInt(actual, expected, expectedResult.Operator) - default: - return false - } -} - -func coerceBool(value any) (bool, bool) { - switch typed := value.(type) { - case bool: - return typed, true - case string: - parsed, err := strconv.ParseBool(typed) - if err != nil { - return false, false - } - return parsed, true - default: - return false, false - } -} - -func coerceInt(value any) (int, bool) { - switch typed := value.(type) { - case int: - return typed, true - case int64: - if typed > math.MaxInt || typed < math.MinInt { - return 0, false - } - return int(typed), true - case float64: - if math.IsNaN(typed) || math.IsInf(typed, 0) { - return 0, false - } - if math.Trunc(typed) != typed { - return 0, false - } - // MaxInt rounds up as float64 on 64-bit hosts; use an exclusive upper bound. - if typed >= -float64(math.MinInt) || typed < float64(math.MinInt) { - return 0, false - } - return int(typed), true - case json.Number: - parsed, ok := new(big.Rat).SetString(typed.String()) - if !ok || !parsed.IsInt() || !parsed.Num().IsInt64() { - return 0, false - } - return coerceInt(parsed.Num().Int64()) - case string: - parsed, err := strconv.Atoi(typed) - if err != nil { - return 0, false - } - return parsed, true - default: - return 0, false - } -} - -func compareInt(actual int, expected int, operator api.JqOperator) bool { - switch operator { - case "==": - return actual == expected - case ">": - return actual > expected - case ">=": - return actual >= expected - case "<": - return actual < expected - case "<=": - return actual <= expected - default: - return false - } -} - func localFailure(stepIndex, testIndex int, message string) *api.StructuredErrCLI { return &api.StructuredErrCLI{ErrorMessage: message, FailedStepIndex: stepIndex, FailedTestIndex: testIndex} } From 6924ab381f92be611e0499c96a8cff6003d0f0b0 Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:19:11 -0400 Subject: [PATCH 5/7] centralize shared variable utils --- checks/cli.go | 26 ------------ checks/http.go | 46 --------------------- checks/{captures.go => variables.go} | 62 ++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 72 deletions(-) rename checks/{captures.go => variables.go} (57%) diff --git a/checks/cli.go b/checks/cli.go index d933a6a..88bfa2d 100644 --- a/checks/cli.go +++ b/checks/cli.go @@ -6,7 +6,6 @@ import ( "maps" "os" "os/exec" - "regexp" "runtime" "strings" @@ -122,31 +121,6 @@ func runCLICommandWithOutputLimit( return result } -func parseStdoutVariables(stdout string, vardefs []api.CLICommandStdoutVariable, variables map[string]string) error { - for _, vardef := range vardefs { - if vardef.Name == "" { - return fmt.Errorf("invalid stdout variable configuration") - } - if vardef.Regex == "" { - return fmt.Errorf("invalid stdout variable configuration") - } - re, err := regexp.Compile(vardef.Regex) - if err != nil { - return fmt.Errorf("invalid stdout variable configuration") - } - if re.NumSubexp() != 1 { - return fmt.Errorf("invalid stdout variable configuration") - } - - matches := re.FindStringSubmatch(stdout) - if len(matches) == 2 { - variables[vardef.Name] = matches[1] - } - } - - return nil -} - func prettyPrintCLICommand(test api.CLICommandTest, variables map[string]string) string { var descriptions []string if test.ExitCode != nil { diff --git a/checks/http.go b/checks/http.go index c472f5d..3466512 100644 --- a/checks/http.go +++ b/checks/http.go @@ -7,7 +7,6 @@ import ( "maps" "net/http" "net/url" - "regexp" "slices" "strings" "unicode/utf8" @@ -21,8 +20,6 @@ const ( maxBinaryBodyBytes = 16 * 1024 ) -var interpolationPattern = regexp.MustCompile(`\$\{([^}]+)\}`) - func runHTTPRequest( client *http.Client, baseURL string, @@ -124,27 +121,6 @@ func runHTTPRequest( return result } -func interpolateJSONStrings(value any, variables map[string]string) any { - switch value := value.(type) { - case string: - return InterpolateVariables(value, variables) - case []any: - interpolated := make([]any, len(value)) - for i, item := range value { - interpolated[i] = interpolateJSONStrings(item, variables) - } - return interpolated - case map[string]any: - interpolated := make(map[string]any, len(value)) - for key, item := range value { - interpolated[key] = interpolateJSONStrings(item, variables) - } - return interpolated - default: - return value - } -} - func prettyPrintHTTPTest(test api.HTTPRequestTest, variables map[string]string) string { var descriptions []string if test.StatusCode != nil { @@ -226,28 +202,6 @@ func findHeaderValue(headers map[string]string, key string) (string, bool) { return "", false } -func InterpolateVariables(template string, vars map[string]string) string { - return interpolationPattern.ReplaceAllStringFunc(template, func(m string) string { - // Extract the key from the match, which is in the form ${key} - key := strings.TrimSuffix(strings.TrimPrefix(m, "${"), "}") - if val, ok := vars[key]; ok { - return val - } - return m - }) -} - -func InterpolationNames(template string) []string { - matches := interpolationPattern.FindAllStringSubmatch(template, -1) - names := make([]string, 0, len(matches)) - for _, match := range matches { - if len(match) > 1 { - names = append(names, match[1]) - } - } - return names -} - func likelyBinary(b []byte) bool { if len(b) == 0 { return false diff --git a/checks/captures.go b/checks/variables.go similarity index 57% rename from checks/captures.go rename to checks/variables.go index 8b5b890..bf9fe44 100644 --- a/checks/captures.go +++ b/checks/variables.go @@ -4,10 +4,72 @@ import ( "errors" "fmt" "regexp" + "strings" api "github.com/bootdotdev/bootdev/client" ) +var interpolationPattern = regexp.MustCompile(`\$\{([^}]+)\}`) + +func InterpolateVariables(template string, vars map[string]string) string { + return interpolationPattern.ReplaceAllStringFunc(template, func(m string) string { + // Extract the key from the match, which is in the form ${key} + key := strings.TrimSuffix(strings.TrimPrefix(m, "${"), "}") + if val, ok := vars[key]; ok { + return val + } + return m + }) +} + +func InterpolationNames(template string) []string { + matches := interpolationPattern.FindAllStringSubmatch(template, -1) + names := make([]string, 0, len(matches)) + for _, match := range matches { + if len(match) > 1 { + names = append(names, match[1]) + } + } + return names +} + +func interpolateJSONStrings(value any, variables map[string]string) any { + switch value := value.(type) { + case string: + return InterpolateVariables(value, variables) + case []any: + interpolated := make([]any, len(value)) + for i, item := range value { + interpolated[i] = interpolateJSONStrings(item, variables) + } + return interpolated + case map[string]any: + interpolated := make(map[string]any, len(value)) + for key, item := range value { + interpolated[key] = interpolateJSONStrings(item, variables) + } + return interpolated + default: + return value + } +} + +func parseStdoutVariables(stdout string, vardefs []api.CLICommandStdoutVariable, variables map[string]string) error { + for _, vardef := range vardefs { + if vardef.Name == "" || vardef.Regex == "" { + return errors.New("invalid stdout variable configuration") + } + value, found, err := regexCapture(vardef.Regex, stdout) + if err != nil { + return errors.New("invalid stdout variable configuration") + } + if found { + variables[vardef.Name] = value + } + } + return nil +} + func parseVariables(body []byte, vardefs []api.HTTPRequestResponseVariable, variables map[string]string) error { bodyString := string(body) for _, vardef := range vardefs { From f32c99edab6e48f6f98259bb8be644538489ea6b Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:30:24 -0400 Subject: [PATCH 6/7] look at actual exit code --- render/view.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render/view.go b/render/view.go index 558a33b..711020f 100644 --- a/render/view.go +++ b/render/view.go @@ -229,8 +229,8 @@ func renderCompactStep(step stepModel, spinner string, isSubmit bool) string { func renderStepResult(step stepModel) string { var str strings.Builder if step.result.CLICommandResult != nil { - for _, test := range step.tests { - if strings.Contains(strings.ToLower(test.text), "exit code") { + for _, test := range step.result.CLICommandResult.Command.Tests { + if test.ExitCode != nil { fmt.Fprintf(&str, "\n > Command exit code: %d\n", step.result.CLICommandResult.ExitCode) break } From 052a96c7086dcc5ee76ecf3f8cb5220e35cd4696 Mon Sep 17 00:00:00 2001 From: Theo Beers Date: Fri, 18 Sep 2026 13:46:14 -0400 Subject: [PATCH 7/7] don't truncate numbers for int assertions --- checks/local.go | 6 +++++- checks/local_test.go | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/checks/local.go b/checks/local.go index f78851b..e5ab555 100644 --- a/checks/local.go +++ b/checks/local.go @@ -248,7 +248,11 @@ func jsonValOp(test api.HTTPRequestTestJSONValue, jsn string, variables map[stri case intOk: v = vInt case floatOk: - v = int(vFloat) + var ok bool + v, ok = coerceInt(vFloat) + if !ok { + return errors.New("expected int value") + } default: return errors.New("expected int value") } diff --git a/checks/local_test.go b/checks/local_test.go index f0b1537..30c6924 100644 --- a/checks/local_test.go +++ b/checks/local_test.go @@ -313,3 +313,26 @@ func TestEvaluateStdoutJqResultTypes(t *testing.T) { }) } } + +func TestHTTPIntegerAssertions(t *testing.T) { + expected := 3 + for _, tt := range []struct { + body string + operator api.OperatorType + pass bool + }{ + {`3`, api.OpEquals, true}, + {`3.0`, api.OpEquals, true}, + {`3.9`, api.OpEquals, false}, + {`4`, api.OpGreaterThan, true}, + {`4.1`, api.OpGreaterThan, false}, + {`"3"`, api.OpEquals, false}, + {`1e20`, api.OpGreaterThan, false}, + } { + assertion := api.HTTPRequestTestJSONValue{Path: ".", Operator: tt.operator, IntValue: &expected} + err := jsonValOp(assertion, tt.body, nil) + if (err == nil) != tt.pass { + t.Errorf("%s %s %d: error = %v, want pass = %t", tt.body, tt.operator, expected, err, tt.pass) + } + } +}