From 19e58577fdb115e8466ccfebf83a2e4c7b3fc938 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sun, 1 Nov 2026 12:03:36 -0800 Subject: [PATCH 1/2] devops(bots): publish the agent's GitHub writes as typed calls The triage and revision workflows switch to the publish-calls action of microsoft/playwright. The agent leaves what must happen on GitHub as typed calls in calls.json, and one job checks and makes them with the app token. It replaces publish-patch and triage-comment. Depends on: https://github.com/microsoft/playwright/pull/43838 --- .claude/skills/playwright-triage/SKILL.md | 38 ++++--- .github/workflows/revision.yml | 26 ++--- .github/workflows/triage.yml | 126 +++++++++------------- 3 files changed, 92 insertions(+), 98 deletions(-) diff --git a/.claude/skills/playwright-triage/SKILL.md b/.claude/skills/playwright-triage/SKILL.md index a81efd618..9b1010f39 100644 --- a/.claude/skills/playwright-triage/SKILL.md +++ b/.claude/skills/playwright-triage/SKILL.md @@ -69,18 +69,32 @@ file, and delete the file when you are done, so that the checkout is clean. language). The gap is in the docs, so it is an upstream change too. Do not edit a generated file, and do not add to `scripts/codegen/python_rules.js` to go around the docs. -Create `$OUTPUT_DIR/upstream/` with: - -- `report.md`: the report for an engineer of `microsoft/playwright` who did not read the Python - issue: what fails, expected versus actual, the versions (the Python package, the commit in - `DRIVER_VERSION`, the browser, the platform), the root cause when you found it, and the condensed - Node test as the repro. -- `report.json`: `{"title": "..."}`, at most 256 characters, for example - `route.fulfill() drops the charset of Content-Type`. - -The comment job appends the title and the body to your comment, for a maintainer to file in -`microsoft/playwright`. The bot never files it. Write `report.json` last, when `report.md` and the -comment are done. Like `proposal.json`, it marks the report as ready. +The report is for an engineer of `microsoft/playwright` who did not read the Python issue: what +fails, expected versus actual, the versions (the Python package, the commit in `DRIVER_VERSION`, the +browser, the platform), the root cause when you found it, and the condensed Node test as the repro. +The title has at most 256 characters, for example `route.fulfill() drops the charset of +Content-Type`. + +The bot never files it in `microsoft/playwright`: this run allows no `issue` call (the calls are +in `$AGENT_GUIDES_DIR/calls.d.ts`). Put the report at the end of your comment, for a maintainer to +file, in this form: + +`````markdown +Upstream bug report, for a maintainer to file in microsoft/playwright: + +**Title:** + +<details> +<summary>Body</summary> + +````markdown +<the report> +```` + +</details> +````` + +Write the rest of the comment as the note: the verdict, the evidence, and that the bug is upstream. ## PRs from third parties diff --git a/.github/workflows/revision.yml b/.github/workflows/revision.yml index 7b4ee534b..8b5befa12 100644 --- a/.github/workflows/revision.yml +++ b/.github/workflows/revision.yml @@ -1,8 +1,9 @@ # Copilot addresses the maintainers' review comments on a draft PR that the triage workflow opened, -# without write credentials. A fresh job pushes the new commit to the PR's branch using the app, so -# CI runs on it. The steps that are the same for every Playwright repo are actions in -# microsoft/playwright, and the shared guide they carry says how to revise; the prompt below has -# what is specific to this repo. +# without write credentials. The agent leaves the calls to make on GitHub in calls.json. A fresh job +# checks them and makes them using the app: it pushes the new commit to the PR's branch, so CI runs +# on it, and replies in the review threads. The steps that are the same for every Playwright repo +# are actions in microsoft/playwright, and the shared guide they carry says how to revise; the +# prompt below has what is specific to this repo. name: PR revision on: @@ -97,14 +98,14 @@ jobs: Pull request: https://github.com/microsoft/playwright-python/pull/${{ github.event.pull_request.number }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: commit/commit.json + result-file: calls.json artifact-name: revision exec: ${{ env.CI_ENV_EXEC }} push: name: Push the revision needs: [prepare, revise] - # commit.json is written last, so a finished revision is pushed even when the job failed later. + # calls.json is written when the revision is done, so it is pushed even if the job failed later. if: ${{ !cancelled() && needs.revise.outputs.has_revision == 'true' }} runs-on: ubuntu-latest steps: @@ -113,6 +114,8 @@ jobs: with: app-id: ${{ vars.PLAYWRIGHT_APP_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.prepare.outputs.head }} @@ -121,12 +124,9 @@ jobs: with: name: revision path: ${{ runner.temp }}/revision - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate and push - uses: microsoft/playwright/.github/actions/publish-patch@main + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + uses: microsoft/playwright/.github/actions/publish-calls@main with: token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/revision/commit - patch: revision.patch - metadata: commit.json - branch: ${{ needs.prepare.outputs.branch }} + dir: ${{ runner.temp }}/revision diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index 064e213be..6e10f1a81 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -1,8 +1,9 @@ # Copilot triages and fixes an issue, or works on a task given by hand, without write credentials. -# Fresh jobs post the run link on the issue, open the completed fix as a draft PR using the app, so -# CI runs on it, and comment on the issue. The steps that are the same for every Playwright repo are -# actions in microsoft/playwright. The skill in .claude/skills/playwright-triage/ has what is -# specific to this repo. +# The agent leaves the calls to make on GitHub in calls.json. A fresh job posts the run link on the +# issue first. After the agent, a fresh job checks the calls and makes them using the app: it opens +# the fix as a draft PR, so CI runs on it, and comments on the issue. The steps that are the same +# for every Playwright repo are actions in microsoft/playwright. The skill in +# .claude/skills/playwright-triage/ has what is specific to this repo. name: Issue triage on: @@ -86,7 +87,6 @@ jobs: base: ${{ steps.base.outputs.sha }} setup: ${{ steps.setup.outcome }} agent: ${{ steps.agent.outputs.outcome }} - has_fix: ${{ steps.agent.outputs.has-result }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -130,98 +130,78 @@ jobs: with: prompt: ${{ steps.prompt.outputs.prompt }} output-dir: ${{ env.OUTPUT_DIR }} - result-file: fix/proposal.json + result-file: calls.json artifact-name: triage github-tools: true exec: ${{ env.CI_ENV_EXEC }} - - name: Check the upstream bug report - run: | - if [[ -f "$OUTPUT_DIR/upstream/report.json" ]]; then - title=$(node -p 'String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).title ?? "")' "$OUTPUT_DIR/upstream/report.json" || true) - if [[ -z "$title" ]]; then - echo "::warning::upstream/report.json has no title, so the report is not appended to the comment" - else - echo "Upstream bug report: $title" >> "$GITHUB_STEP_SUMMARY" - fi - fi publish: - name: Open draft pull request - needs: triage - # proposal.json is written last, so a finished proposal is published even when the job failed later. - if: ${{ !cancelled() && needs.triage.outputs.has_fix == 'true' }} + name: Publish the results + needs: [start, triage] + # Runs after every triage run, also a failed one: without a comment from the agent, the comment + # says why the bot could not finish. + if: ${{ !cancelled() && needs.triage.result != 'skipped' }} runs-on: ubuntu-latest - outputs: - pr: ${{ steps.publish.outputs.pull-request }} steps: - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 id: app-token with: app-id: ${{ vars.PLAYWRIGHT_APP_ID }} private-key: ${{ secrets.PLAYWRIGHT_PRIVATE_KEY }} + permission-contents: write + permission-issues: write + permission-pull-requests: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ needs.triage.outputs.base }} token: ${{ steps.app-token.outputs.token }} - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: triage - path: ${{ runner.temp }}/triage - # Never execute code or scripts from the proposed changes in the app job. - - name: Validate, push and open draft - id: publish - uses: microsoft/playwright/.github/actions/publish-patch@main - with: - token: ${{ steps.app-token.outputs.token }} - dir: ${{ runner.temp }}/triage/fix - patch: fix.patch - metadata: proposal.json - body-file: pr-body.md - branch: triage/${{ github.event.issue.number || inputs.issue || 'task' }}/${{ github.run_id }}-${{ github.run_attempt }} - commit-body: "${{ (github.event.issue.number || inputs.issue) && format('Fixes: https://github.com/{0}/issues/{1}', github.repository, github.event.issue.number || inputs.issue) || '' }}" - pull-request: true - - comment: - name: Comment on the issue - needs: [start, triage, publish] - # Runs after every triage run that has an issue, including failed ones, and whatever publish did. - if: ${{ !cancelled() && needs.triage.result != 'skipped' && (github.event_name == 'issues' || inputs.issue != '') }} - runs-on: ubuntu-latest - permissions: - issues: write - steps: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 # A failed triage run may not have uploaded anything. continue-on-error: true with: name: triage path: ${{ runner.temp }}/triage - # The bot never files upstream; the comment hands the report to a maintainer. - - name: Describe the upstream bug report - id: upstream + # Without a comment from the agent on the issue, for example after a failed run, the issue gets + # one that says that the bot could not finish. + - name: Add a comment when the agent left none + if: github.event.issue.number || inputs.issue env: - OUTPUT_DIR: ${{ runner.temp }}/triage + CALLS: ${{ runner.temp }}/triage/calls.json + ISSUE: ${{ github.event.issue.number || inputs.issue }} + SETUP: ${{ needs.triage.outputs.setup }} + AGENT: ${{ needs.triage.outputs.agent }} + FOOTER: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | - extra="" - if [[ -s "$OUTPUT_DIR/upstream/report.md" && -f "$OUTPUT_DIR/upstream/report.json" ]]; then - title=$(jq -r '.title // empty' "$OUTPUT_DIR/upstream/report.json") - if [[ -n "$title" ]]; then - extra=$(printf 'Upstream bug report, for a maintainer to file in microsoft/playwright:\n\n**Title:** %s\n\n<details>\n<summary>Body</summary>\n\n````markdown\n%s\n````\n\n</details>' "$title" "$(cat "$OUTPUT_DIR/upstream/report.md")") - fi + mkdir -p "$(dirname "$CALLS")" + if jq -e --argjson issue "$ISSUE" 'any(.[]; .call == "comment" and .issue == $issue)' "$CALLS" > /dev/null 2>&1; then + exit 0 fi - delimiter="EXTRA_$RANDOM$RANDOM" - { echo "extra<<$delimiter"; echo "$extra"; echo "$delimiter"; } >> "$GITHUB_OUTPUT" - - name: Post the triage comment - uses: microsoft/playwright/.github/actions/triage-comment@main + if [[ "$SETUP" == "failure" ]]; then + reason="setting up the runner failed" + elif [[ "$AGENT" == "failure" ]]; then + reason="the Copilot session failed or ran out of budget" + else + reason="the session ended without a comment" + fi + body=$(printf "Hi, I'm the Playwright bot. I couldn't finish triaging this issue: %s. The agent run has the details.\n\n<sub>%s - [agent run](%s)</sub>\n<!-- playwright-ai-triage -->" "$reason" "$FOOTER" "$RUN_URL") + comment=$(jq -n --arg repo "$GITHUB_REPOSITORY" --argjson issue "$ISSUE" --arg body "$body" '{call: "comment", repo: $repo, issue: $issue, body: $body}') + # A calls.json that is not an array is replaced: its calls would be rejected anyway. + jq --argjson comment "$comment" '. + [$comment]' "$CALLS" > "$CALLS.tmp" 2> /dev/null \ + || jq -n --argjson comment "$comment" '[$comment]' > "$CALLS.tmp" + mv "$CALLS.tmp" "$CALLS" + # Never execute code or scripts from the artifact in the app job. The calls are data. + - name: Check and make the calls + id: publish + uses: microsoft/playwright/.github/actions/publish-calls@main with: - issue: ${{ github.event.issue.number || inputs.issue }} + token: ${{ steps.app-token.outputs.token }} dir: ${{ runner.temp }}/triage - footer: Triaged on ${{ fromJSON('{"linux":"Linux","mac":"macOS","windows":"Windows","needs-triage":"Linux","needs-triage-mac":"macOS","needs-triage-win":"Windows"}')[inputs.platform || github.event.label.name] }} by the Playwright bot - job-result: ${{ needs.triage.result }} - setup-outcome: ${{ needs.triage.outputs.setup }} - agent-outcome: ${{ needs.triage.outputs.agent }} - setup-failure-reason: setting up the runner, building the project or downloading the browsers failed - pull-request: ${{ needs.publish.outputs.pr }} - publish-result: ${{ needs.publish.result }} - extra: ${{ steps.upstream.outputs.extra }} - start-comment: ${{ needs.start.outputs.comment }} + - name: Hide the comment with the run link + if: ${{ !cancelled() && steps.publish.outputs.comment == 'true' && needs.start.outputs.comment != '' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + ID: ${{ needs.start.outputs.comment }} + run: | + gh api graphql -f query='mutation($id: ID!) { minimizeComment(input: { subjectId: $id, classifier: OUTDATED }) { clientMutationId } }' -f id="$ID" \ + || echo "::warning::Could not hide the comment with the run link" From 65ffa7e10762b9381ea4edc37a4c71b59428058c Mon Sep 17 00:00:00 2001 From: Pavel Feldman <pavel.feldman@gmail.com> Date: Sun, 1 Nov 2026 14:08:01 -0800 Subject: [PATCH 2/2] devops(bots): let collect-review read the issue of the PR --- .github/workflows/revision.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/revision.yml b/.github/workflows/revision.yml index 8b5befa12..e0f7730d8 100644 --- a/.github/workflows/revision.yml +++ b/.github/workflows/revision.yml @@ -25,6 +25,7 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest permissions: + issues: read pull-requests: write env: PLATFORM: ${{ fromJSON('{"needs-revision":"linux","needs-revision-mac":"mac","needs-revision-win":"windows"}')[github.event.label.name] }}