Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions apps/stackpanel-go/cmd/cli/setup_agent_reply.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,19 @@ func runSetupAgentReply(ctx context.Context, agent setupagent.Agent, phase setup
if pending.Phase != phase {
return nil, fmt.Errorf("saved reply belongs to %s, not %s; use --restart to review a new plan", pending.Phase, phase)
}
ui.Warning(fmt.Sprintf("%s returned malformed setup JSON. Requesting a corrected reply · retry 1 of 1. Your choices are saved.", agent.ID))
request.ReadOnly = true
request.Prompt = setupagent.ReplyCorrectionPrompt(phase, pending.Message, pending.Problem)
result, err := setupagent.Run(ctx, agent, request)
reply, err := setupagent.ParseReply(pending.Message)
if err != nil {
return nil, fmt.Errorf("%s reply correction failed; original reply saved for retry: %w", agent.ID, err)
}
reply, err := setupagent.ParseReply(result.Message)
if err != nil {
return nil, fmt.Errorf("%s returned an invalid reply after one formatting retry; original reply saved: %w", agent.ID, err)
ui.Warning(fmt.Sprintf("%s returned malformed setup JSON. Requesting a corrected reply · retry 1 of 1. Your choices are saved.", agent.ID))
request.ReadOnly = true
request.Prompt = setupagent.ReplyCorrectionPrompt(phase, pending.Message, pending.Problem)
result, err := setupagent.Run(ctx, agent, request)
if err != nil {
return nil, fmt.Errorf("%s reply correction failed; original reply saved for retry: %w", agent.ID, err)
}
reply, err = setupagent.ParseReply(result.Message)
if err != nil {
return nil, fmt.Errorf("%s returned an invalid reply after one formatting retry; original reply saved: %w", agent.ID, err)
}
}
if reply.Status == "blocked" {
return nil, fmt.Errorf("%s could not recover its reply: %s", agent.ID, reply.Summary)
Expand Down
103 changes: 101 additions & 2 deletions apps/stackpanel-go/cmd/cli/setup_agent_reply_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,14 @@ func TestSetupReplyCorrectionResumesWithoutRepeatingWork(t *testing.T) {
if phase == setupagent.Inspection {
message = `{"status":"plan","plan":{"summary":"Create app","expectations":{"version":1,"files":["src/main.ts"],"config":[{"path":["enable"],"equals":true}],"commands":[{"id":"build","scope":"build","dir":".","argv":["bun","run","build"]}]}}}`
}
malformed := strings.TrimSuffix(message, "}") // The user's missing outer brace.
// Missing content still requires a model correction. A missing final
// outer brace alone is recovered without another invocation.
malformed := strings.TrimSuffix(message, "}")
if phase == setupagent.Inspection {
malformed = strings.TrimSuffix(malformed, "}")
} else {
malformed = strings.TrimSuffix(malformed, `"`)
}
writeSetupReplyEvents(t, fixture, "original", malformed)
writeSetupReplyEvents(t, fixture, "correction", malformed)
state := &setupManifest{Version: 1, Root: root, Stage: "apply", path: setupStatePath(root),
Expand Down Expand Up @@ -126,7 +133,7 @@ func TestSetupReplyDoesNotRetryProviderFailure(t *testing.T) {

func TestSetupReplyCorrectionCannotReplaceAcceptedPlan(t *testing.T) {
agent, fixture := setupReplyFixture(t)
writeSetupReplyEvents(t, fixture, "original", `{"status":"complete","summary":"Done"`)
writeSetupReplyEvents(t, fixture, "original", `{"status":"complete","summary":"Done`)
writeSetupReplyEvents(t, fixture, "correction", `{"status":"plan","plan":{"summary":"Weaken checks","expectations":{"version":1,"config":[{"path":["enable"],"equals":true}]}}}`)
var output bytes.Buffer
ui := tui.NewSetupUI(context.Background(), false, &output)
Expand All @@ -137,3 +144,95 @@ func TestSetupReplyCorrectionCannotReplaceAcceptedPlan(t *testing.T) {
t.Fatalf("accepted replacement plan: %v", err)
}
}

func TestSetupReplyMissingFinalOuterBraceDoesNotRetry(t *testing.T) {
for _, phase := range []setupagent.Phase{setupagent.Inspection, setupagent.Setup} {
t.Run(string(phase), func(t *testing.T) {
agent, fixture := setupReplyFixture(t)
message := `{"status":"complete","summary":"Created agreed files"}`
if phase == setupagent.Inspection {
message = `{"status":"plan","plan":{"summary":"Configure app","expectations":{"version":1,"config":[{"path":["enable"],"equals":true}]}}}`
}
writeSetupReplyEvents(t, fixture, "original", strings.TrimSuffix(message, "}"))
var output bytes.Buffer
ui := tui.NewSetupUI(context.Background(), false, &output)
defer ui.Close()
state := &setupManifest{}
reply, err := runSetupAgentReply(context.Background(), agent, phase, setupagent.RunRequest{
Dir: t.TempDir(), ReadOnly: phase == setupagent.Inspection,
}, state, ui)
if err != nil || reply == nil || state.PendingReply != nil {
t.Fatalf("recover reply: %+v, state %+v, error %v", reply, state.PendingReply, err)
}
if strings.Contains(output.String(), "malformed") {
t.Fatal("recoverable reply displayed a malformed JSON warning")
}
if _, err := os.Stat(filepath.Join(fixture, "correction-calls")); !os.IsNotExist(err) {
t.Fatalf("recoverable reply invoked correction: %v", err)
}
})
}
}

func TestSetupSavedReplyMissingFinalOuterBraceDoesNotRetry(t *testing.T) {
for _, phase := range []setupagent.Phase{setupagent.Inspection, setupagent.Setup, setupagent.Repair} {
t.Run(string(phase), func(t *testing.T) {
setupStateTestEnvironment(t)
agent, fixture := setupReplyFixture(t)
root := t.TempDir()
message := `{"status":"complete","summary":"Created agreed files"}`
if phase == setupagent.Inspection {
message = `{"status":"plan","plan":{"summary":"Configure app","expectations":{"version":1,"config":[{"path":["enable"],"equals":true}]}}}`
}
state := &setupManifest{Version: 1, Root: root, Stage: "apply", path: setupStatePath(root),
Plan: &setupagent.Plan{Expectations: reconcile.Expectations{Version: 1, RequiredChecks: []string{"keep-check"}}},
Request: setupagent.SetupRequest{Root: root, Answers: []setupagent.Answer{{ID: "framework", Values: []string{"Bun"}}}},
PendingReply: &setupReplyRecovery{Phase: phase, Message: strings.TrimSuffix(message, "}"), Problem: "invalid setup reply: unexpected EOF"},
}
if err := state.save(); err != nil {
t.Fatal(err)
}
loaded, err := loadSetupManifest(root)
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
ui := tui.NewSetupUI(context.Background(), false, &output)
defer ui.Close()
reply, err := runSetupAgentReply(context.Background(), agent, phase, setupagent.RunRequest{
Dir: root, ReadOnly: phase == setupagent.Inspection,
}, loaded, ui)
if err != nil || reply == nil {
t.Fatalf("recover saved reply: %+v, error %v", reply, err)
}
saved, err := loadSetupManifest(root)
if err != nil || saved.PendingReply != nil || len(saved.Request.Answers) != 1 ||
saved.Plan == nil || len(saved.Plan.Expectations.RequiredChecks) != 1 || saved.Plan.Expectations.RequiredChecks[0] != "keep-check" {
t.Fatalf("lost saved state: %+v, error %v", saved, err)
}
if strings.Contains(output.String(), "malformed") {
t.Fatal("recoverable saved reply displayed a malformed JSON warning")
}
for _, name := range []string{"original-calls", "correction-calls"} {
if _, err := os.Stat(filepath.Join(fixture, name)); !os.IsNotExist(err) {
t.Fatalf("recoverable saved reply invoked %s: %v", name, err)
}
}
})
}
}

func TestSetupSavedReplyCannotReplaceAcceptedPlan(t *testing.T) {
agent, _ := setupReplyFixture(t)
var output bytes.Buffer
ui := tui.NewSetupUI(context.Background(), false, &output)
defer ui.Close()
state := &setupManifest{PendingReply: &setupReplyRecovery{
Phase: setupagent.Setup,
Message: `{"status":"plan","plan":{"summary":"Weaken checks","expectations":{"version":1,"config":[{"path":["enable"],"equals":true}]}}`,
}}
_, err := runSetupAgentReply(context.Background(), agent, setupagent.Setup, setupagent.RunRequest{Dir: t.TempDir()}, state, ui)
if err == nil || !strings.Contains(err.Error(), "unexpected status") || state.PendingReply == nil {
t.Fatalf("accepted replacement plan from saved reply: %v", err)
}
}
12 changes: 11 additions & 1 deletion apps/stackpanel-go/internal/setupagent/prompt.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package setupagent

import (
"encoding/json"
"errors"
"fmt"
"io"
"path/filepath"
Expand Down Expand Up @@ -101,7 +102,16 @@ func ParseReply(message string) (*Reply, error) {
return nil, fmt.Errorf("agent reply exceeds size limit")
}
var reply Reply
if err := decodeStrict(message, &reply); err != nil {
err := decodeStrict(message, &reply)
if errors.Is(err, io.ErrUnexpectedEOF) && len(message) < maxMessageBytes {
// Completed provider replies can omit their final outer brace. Recover
// only that delimiter; incomplete nested content still fails decoding.
var recovered Reply
if decodeStrict(message+"}", &recovered) == nil {
reply, err = recovered, nil
}
}
if err != nil {
// Accept the original inspection protocol during the experiment's transition.
if plan, planErr := ParsePlan(message); planErr == nil {
return &Reply{Status: "plan", Plan: plan}, nil
Expand Down
62 changes: 62 additions & 0 deletions apps/stackpanel-go/internal/setupagent/reply_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package setupagent

import (
"context"
"reflect"
"strings"
"testing"
)

func TestRunRecoversReplyMissingFinalOuterBrace(t *testing.T) {
plan := `{"status":"plan","plan":` + validPlan + `}`
for _, tt := range []struct {
name, message string
readOnly bool
}{
{"inspection", plan, true},
{"setup", complete, false},
} {
t.Run(tt.name, func(t *testing.T) {
agent := fakeAgent(t, "codex")
// The provider completed normally, but omitted only the reply's
// outermost closing brace. Its nested plan remains complete.
t.Setenv("STACKPANEL_TEST_AGENT_EVENTS", codexEvents(strings.TrimSuffix(tt.message, "}")))
result, err := Run(context.Background(), agent, RunRequest{Dir: t.TempDir(), ReadOnly: tt.readOnly})
if err != nil {
t.Fatal(err)
}
actual, err := ParseReply(result.Message)
if err != nil {
t.Fatalf("completed provider reply was rejected: %v", err)
}
expected, err := ParseReply(tt.message)
if err != nil || !reflect.DeepEqual(actual, expected) {
t.Fatalf("recovery changed reply: got %+v, want %+v (%v)", actual, expected, err)
}
})
}
}

func TestReplyRecoveryRejectsIncompleteOrInvalidContents(t *testing.T) {
for _, tt := range []struct{ name, message string }{
{"nested object", `{"status":"plan","plan":` + strings.TrimSuffix(validPlan, "}")},
{"nested array", `{"status":"needs_input","questions":[{"id":"framework","prompt":"Framework?","kind":"text","required":true}`},
{"unfinished string", `{"status":"complete","summary":"Done`},
{"missing value", `{"status":"complete","summary":`},
{"trailing comma", `{"status":"complete","summary":"Done",`},
{"unknown field", `{"status":"complete","summary":"Done","unexpected":true`},
{"invalid plan", `{"status":"plan","plan":{"summary":"No checks","expectations":{"version":1,"config":[]}}`},
} {
t.Run(tt.name, func(t *testing.T) {
agent := fakeAgent(t, "codex")
t.Setenv("STACKPANEL_TEST_AGENT_EVENTS", codexEvents(tt.message))
result, err := Run(context.Background(), agent, RunRequest{Dir: t.TempDir(), ReadOnly: true})
if err != nil {
t.Fatal(err)
}
if _, err := ParseReply(result.Message); err == nil {
t.Fatal("accepted incomplete or invalid reply")
}
})
}
}
Loading