feat: issue triage tooling (#698)
* feat: issue triage tooling * fix: update triage-new-issues script Co-authored-by: Dale Seo <5466341+DaleSeo@users.noreply.github.com> --------- Co-authored-by: Dale Seo <5466341+DaleSeo@users.noreply.github.com>
This commit is contained in:
Alex Hancock
GitHub
parent
d6703dad75
commit
98653855ef
2 changed files with 473 additions and 0 deletions
73
.github/workflows/triage.yml
vendored
Normal file
73
.github/workflows/triage.yml
vendored
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
name: Auto Triage Issues
|
||||
|
||||
on:
|
||||
# Triage newly opened or reopened issues immediately
|
||||
issues:
|
||||
types: [opened, reopened]
|
||||
|
||||
# Daily sweep to catch anything missed (e.g., label removals, edits)
|
||||
schedule:
|
||||
- cron: "0 9 * * *" # 9:00 UTC daily
|
||||
|
||||
# Allow manual runs from the Actions tab
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
issue_number:
|
||||
description: "Triage a specific issue number (leave empty for all untriaged)"
|
||||
required: false
|
||||
type: string
|
||||
dry_run:
|
||||
description: "Dry-run mode (preview only, don't apply labels)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
triage:
|
||||
name: Triage Issues
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL || 'gpt-4o-mini' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install jq
|
||||
run: sudo apt-get install -y jq
|
||||
|
||||
- name: Triage single issue (on issue event)
|
||||
if: github.event_name == 'issues'
|
||||
run: |
|
||||
./scripts/triage-new-issues.sh \
|
||||
--issue ${{ github.event.issue.number }} \
|
||||
--apply
|
||||
|
||||
- name: Triage specific issue (manual dispatch)
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch'
|
||||
&& github.event.inputs.issue_number != ''
|
||||
run: |
|
||||
ARGS=(--issue ${{ github.event.inputs.issue_number }})
|
||||
if [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then
|
||||
ARGS+=(--apply)
|
||||
fi
|
||||
./scripts/triage-new-issues.sh "${ARGS[@]}"
|
||||
|
||||
- name: Triage all untriaged issues (schedule or manual sweep)
|
||||
if: >-
|
||||
github.event_name == 'schedule'
|
||||
|| (github.event_name == 'workflow_dispatch'
|
||||
&& github.event.inputs.issue_number == '')
|
||||
run: |
|
||||
ARGS=()
|
||||
if [[ "${{ github.event.inputs.dry_run }}" != "true" ]]; then
|
||||
ARGS+=(--apply)
|
||||
fi
|
||||
./scripts/triage-new-issues.sh "${ARGS[@]}"
|
||||
400
scripts/triage-new-issues.sh
Executable file
400
scripts/triage-new-issues.sh
Executable file
|
|
@ -0,0 +1,400 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# triage-new-issues.sh — Ongoing Issue Triage for modelcontextprotocol/rust-sdk
|
||||
#
|
||||
# Finds open issues that are missing required triage labels (type + priority)
|
||||
# and uses an LLM to classify them automatically.
|
||||
#
|
||||
# Modes:
|
||||
# Single issue: ./scripts/triage-new-issues.sh --issue 700
|
||||
# All untriaged: ./scripts/triage-new-issues.sh
|
||||
# Apply labels: ./scripts/triage-new-issues.sh --apply
|
||||
# Both: ./scripts/triage-new-issues.sh --issue 700 --apply
|
||||
#
|
||||
# Environment:
|
||||
# OPENAI_API_KEY — Required. API key for the LLM (OpenAI-compatible endpoint)
|
||||
# OPENAI_BASE_URL — Optional. Override the API base URL (default: https://api.openai.com/v1)
|
||||
# TRIAGE_MODEL — Optional. Model to use (default: gpt-4o-mini)
|
||||
# GITHUB_TOKEN — Optional. Used by `gh` CLI for GitHub API access
|
||||
#
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
REPO="modelcontextprotocol/rust-sdk"
|
||||
DRY_RUN=true
|
||||
SINGLE_ISSUE=""
|
||||
MODEL="${TRIAGE_MODEL:-gpt-4o-mini}"
|
||||
BASE_URL="${OPENAI_BASE_URL:-https://api.openai.com/v1}"
|
||||
TRIAGED=0
|
||||
SKIPPED=0
|
||||
FAILED=0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse arguments
|
||||
# ---------------------------------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--apply) DRY_RUN=false; shift ;;
|
||||
--issue) SINGLE_ISSUE="$2"; shift 2 ;;
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--apply] [--issue NUMBER] [--model MODEL]"
|
||||
echo ""
|
||||
echo " --apply Apply labels to GitHub (default: dry-run)"
|
||||
echo " --issue NUM Triage a single issue by number"
|
||||
echo " --model MODEL LLM model to use (default: gpt-4o-mini)"
|
||||
echo ""
|
||||
echo "Environment:"
|
||||
echo " OPENAI_API_KEY Required. API key for the LLM"
|
||||
echo " OPENAI_BASE_URL Optional. API base URL"
|
||||
echo " TRIAGE_MODEL Optional. Model override"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown argument: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preflight checks
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! command -v gh &>/dev/null; then
|
||||
echo "Error: 'gh' CLI is required. Install from https://cli.github.com/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v jq &>/dev/null; then
|
||||
echo "Error: 'jq' is required. Install with: brew install jq"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
|
||||
echo "Error: OPENAI_API_KEY environment variable is required."
|
||||
echo "Set it to an OpenAI API key, or set OPENAI_BASE_URL for a compatible endpoint."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "============================================="
|
||||
echo " rust-sdk Ongoing Issue Triage"
|
||||
echo " Repo: $REPO"
|
||||
echo " Model: $MODEL"
|
||||
if $DRY_RUN; then
|
||||
echo " Mode: DRY-RUN (pass --apply to execute)"
|
||||
else
|
||||
echo " Mode: APPLYING CHANGES"
|
||||
fi
|
||||
echo "============================================="
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Label definitions — used to build the LLM prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
TYPE_LABELS='["bug", "enhancement", "question"]'
|
||||
PRIORITY_LABELS='["P0", "P1", "P2", "P3"]'
|
||||
WORKFLOW_LABELS='["needs confirmation", "needs repro", "ready for work"]'
|
||||
COMPONENT_LABELS='["T-core", "T-transport", "T-macros", "T-handler", "T-model", "T-security", "T-documentation", "T-examples", "T-service", "T-test", "T-CI", "T-config", "T-dependencies"]'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build the system prompt for the LLM
|
||||
# ---------------------------------------------------------------------------
|
||||
read -r -d '' SYSTEM_PROMPT << 'SYSTEM_EOF' || true
|
||||
You are an issue triage bot for the modelcontextprotocol/rust-sdk repository — a Rust implementation of the Model Context Protocol (MCP).
|
||||
|
||||
Your job is to classify GitHub issues by assigning labels. You MUST return valid JSON with exactly these fields:
|
||||
|
||||
{
|
||||
"type": "<one of: bug, enhancement, question>",
|
||||
"priority": "<one of: P0, P1, P2, P3>",
|
||||
"components": ["<zero or more from the component list>"],
|
||||
"workflow": "<one of: needs confirmation, needs repro, ready for work, or null>",
|
||||
"reasoning": "<one sentence explaining your classification>"
|
||||
}
|
||||
|
||||
## Label Definitions
|
||||
|
||||
### Type
|
||||
- bug: Something is not working (errors, crashes, incorrect behavior)
|
||||
- enhancement: New feature or improvement request
|
||||
- question: User asking for help or clarification
|
||||
|
||||
### Priority
|
||||
- P0: Critical — blocking, security vulnerability, data loss, or crash affecting all users
|
||||
- P1: High — MCP spec violation, conformance blocker, or significant functionality broken
|
||||
- P2: Medium — important but non-blocking improvement, interop issue, or DX gap
|
||||
- P3: Low — nice-to-have, exploratory, long-term, or questions
|
||||
|
||||
### Components (prefix: T-)
|
||||
- T-core: Core library (rmcp crate internals, JSON-RPC, error handling)
|
||||
- T-transport: Transport layer (stdio, SSE, streamable HTTP)
|
||||
- T-macros: Proc macros (#[tool], #[prompt], etc.)
|
||||
- T-handler: Handler/service implementation
|
||||
- T-model: Model/data structures and JSON-RPC types
|
||||
- T-security: OAuth, auth, security features
|
||||
- T-documentation: Documentation and guides
|
||||
- T-examples: Example code
|
||||
- T-service: Service layer
|
||||
- T-test: Testing
|
||||
- T-CI: CI/CD workflows
|
||||
- T-config: Configuration
|
||||
- T-dependencies: Dependency updates
|
||||
|
||||
### Workflow
|
||||
- "needs confirmation": Bug report that needs verification from a maintainer
|
||||
- "needs repro": Bug report without a minimal reproduction case
|
||||
- "ready for work": Issue is well-scoped and ready for a contributor to pick up
|
||||
- null: None of the above apply
|
||||
|
||||
## Rules
|
||||
1. Every issue MUST get exactly one type and one priority.
|
||||
2. Assign 0-2 component labels (only if clearly relevant).
|
||||
3. Assign a workflow label only when appropriate; default to null.
|
||||
4. When in doubt between two priorities, pick the higher one.
|
||||
5. Security issues are always P0.
|
||||
6. MCP spec violations are P1.
|
||||
7. Questions from users are typically P3.
|
||||
8. Return ONLY the JSON object, no markdown fences, no extra text.
|
||||
SYSTEM_EOF
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# classify_issue — call the LLM to classify a single issue
|
||||
# ---------------------------------------------------------------------------
|
||||
classify_issue() {
|
||||
local title="$1"
|
||||
local body="$2"
|
||||
local number="$3"
|
||||
local existing_labels="$4"
|
||||
|
||||
# Truncate body to ~3000 chars to stay within token limits
|
||||
local truncated_body
|
||||
truncated_body="$(echo "$body" | head -c 3000)"
|
||||
|
||||
local user_prompt="Classify this GitHub issue.
|
||||
|
||||
Issue #${number}: ${title}
|
||||
|
||||
Existing labels: ${existing_labels}
|
||||
|
||||
Body:
|
||||
${truncated_body}"
|
||||
|
||||
# Build the JSON payload
|
||||
local payload
|
||||
payload=$(jq -n \
|
||||
--arg model "$MODEL" \
|
||||
--arg system "$SYSTEM_PROMPT" \
|
||||
--arg user "$user_prompt" \
|
||||
'{
|
||||
model: $model,
|
||||
temperature: 0.1,
|
||||
messages: [
|
||||
{ role: "system", content: $system },
|
||||
{ role: "user", content: $user }
|
||||
]
|
||||
}')
|
||||
|
||||
# Call the LLM
|
||||
local response
|
||||
response=$(curl -s -w "\n%{http_code}" \
|
||||
"${BASE_URL}/chat/completions" \
|
||||
-H "Authorization: Bearer ${OPENAI_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$payload" 2>/dev/null)
|
||||
|
||||
local http_code
|
||||
http_code=$(echo "$response" | tail -1)
|
||||
local body_response
|
||||
body_response=$(echo "$response" | sed '$d')
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
echo "ERROR: LLM API returned HTTP $http_code" >&2
|
||||
echo "$body_response" | jq -r '.error.message // .' >&2 2>/dev/null || echo "$body_response" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Extract the content from the response
|
||||
local content
|
||||
content=$(echo "$body_response" | jq -r '.choices[0].message.content' 2>/dev/null)
|
||||
|
||||
if [[ -z "$content" || "$content" == "null" ]]; then
|
||||
echo "ERROR: Empty response from LLM" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Strip markdown fences if present
|
||||
content=$(echo "$content" | sed 's/^```json//; s/^```//; s/```$//' | tr -d '\n')
|
||||
|
||||
# Validate it's valid JSON with required fields
|
||||
if ! echo "$content" | jq -e '.type and .priority' &>/dev/null; then
|
||||
echo "ERROR: LLM returned invalid classification: $content" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$content"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_labels — apply the classification labels to an issue
|
||||
# ---------------------------------------------------------------------------
|
||||
apply_labels() {
|
||||
local issue_num="$1"
|
||||
local classification="$2"
|
||||
|
||||
local type_label priority_label workflow_label reasoning
|
||||
type_label=$(echo "$classification" | jq -r '.type')
|
||||
priority_label=$(echo "$classification" | jq -r '.priority')
|
||||
workflow_label=$(echo "$classification" | jq -r '.workflow // empty')
|
||||
reasoning=$(echo "$classification" | jq -r '.reasoning // "No reasoning provided"')
|
||||
|
||||
# Collect component labels
|
||||
local components
|
||||
components=$(echo "$classification" | jq -r '.components[]? // empty' 2>/dev/null)
|
||||
|
||||
# Build label list
|
||||
local labels=("$type_label" "$priority_label")
|
||||
if [[ -n "$workflow_label" && "$workflow_label" != "null" ]]; then
|
||||
labels+=("$workflow_label")
|
||||
fi
|
||||
while IFS= read -r comp; do
|
||||
[[ -n "$comp" ]] && labels+=("$comp")
|
||||
done <<< "$components"
|
||||
|
||||
# Build gh command
|
||||
local cmd_args=(gh issue edit "$issue_num" --repo "$REPO")
|
||||
for label in "${labels[@]}"; do
|
||||
cmd_args+=(--add-label "$label")
|
||||
done
|
||||
|
||||
echo " Labels: ${labels[*]}"
|
||||
echo " Reasoning: $reasoning"
|
||||
|
||||
if $DRY_RUN; then
|
||||
echo " [DRY-RUN] ${cmd_args[*]}"
|
||||
else
|
||||
echo " [APPLY] Labeling #$issue_num..."
|
||||
if "${cmd_args[@]}" 2>/dev/null; then
|
||||
echo " ✅ Done"
|
||||
else
|
||||
echo " ❌ Failed to apply labels"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# has_triage_labels — check if an issue already has type + priority labels
|
||||
# ---------------------------------------------------------------------------
|
||||
has_triage_labels() {
|
||||
local labels_json="$1"
|
||||
|
||||
local has_type has_priority
|
||||
has_type=$(echo "$labels_json" | jq '[.[] | select(. == "bug" or . == "enhancement" or . == "question")] | length')
|
||||
has_priority=$(echo "$labels_json" | jq '[.[] | select(test("^P[0-3]$"))] | length')
|
||||
|
||||
[[ "$has_type" -gt 0 && "$has_priority" -gt 0 ]]
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# triage_issue — fetch, classify, and label a single issue
|
||||
# ---------------------------------------------------------------------------
|
||||
triage_issue() {
|
||||
local issue_num="$1"
|
||||
|
||||
# Fetch issue details
|
||||
local issue_json
|
||||
issue_json=$(gh issue view "$issue_num" --repo "$REPO" --json title,body,labels 2>/dev/null)
|
||||
|
||||
if [[ -z "$issue_json" ]]; then
|
||||
echo " ❌ Could not fetch issue #$issue_num"
|
||||
FAILED=$((FAILED + 1))
|
||||
return 1
|
||||
fi
|
||||
|
||||
local title body labels_json labels_str
|
||||
title=$(echo "$issue_json" | jq -r '.title')
|
||||
body=$(echo "$issue_json" | jq -r '.body // ""')
|
||||
labels_json=$(echo "$issue_json" | jq '[.labels[].name]')
|
||||
labels_str=$(echo "$labels_json" | jq -r 'join(", ")')
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Issue #$issue_num: $title"
|
||||
echo " Current labels: ${labels_str:-none}"
|
||||
|
||||
# Check if already triaged
|
||||
if has_triage_labels "$labels_json"; then
|
||||
echo " ⏭️ Already triaged (has type + priority). Skipping."
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Classify with LLM
|
||||
echo " 🤖 Classifying with $MODEL..."
|
||||
local classification
|
||||
if ! classification=$(classify_issue "$title" "$body" "$issue_num" "$labels_str"); then
|
||||
echo " ❌ Classification failed"
|
||||
FAILED=$((FAILED + 1))
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Apply labels
|
||||
if apply_labels "$issue_num" "$classification"; then
|
||||
TRIAGED=$((TRIAGED + 1))
|
||||
else
|
||||
FAILED=$((FAILED + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main: single issue or scan all untriaged
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ -n "$SINGLE_ISSUE" ]]; then
|
||||
echo "--- Triaging single issue #$SINGLE_ISSUE ---"
|
||||
echo ""
|
||||
triage_issue "$SINGLE_ISSUE"
|
||||
else
|
||||
echo "--- Scanning for untriaged open issues ---"
|
||||
echo ""
|
||||
|
||||
# Fetch all open issues (paginated, up to 500)
|
||||
issue_numbers=$(gh issue list --repo "$REPO" --state open --limit 500 --json number,labels \
|
||||
| jq -r '.[] | select(
|
||||
([.labels[].name | select(. == "bug" or . == "enhancement" or . == "question")] | length) == 0
|
||||
or
|
||||
([.labels[].name | select(startswith("P"))] | length) == 0
|
||||
) | .number')
|
||||
|
||||
if [[ -z "$issue_numbers" ]]; then
|
||||
echo "✅ All open issues are already triaged! Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
count=$(echo "$issue_numbers" | wc -l | tr -d ' ')
|
||||
echo "Found $count untriaged issue(s)."
|
||||
echo ""
|
||||
|
||||
while IFS= read -r num; do
|
||||
[[ -z "$num" ]] && continue
|
||||
triage_issue "$num"
|
||||
echo ""
|
||||
# Rate-limit: small delay between LLM calls
|
||||
sleep 1
|
||||
done <<< "$issue_numbers"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "============================================="
|
||||
echo " Triage Summary"
|
||||
echo ""
|
||||
echo " Triaged: $TRIAGED"
|
||||
echo " Skipped: $SKIPPED (already triaged)"
|
||||
echo " Failed: $FAILED"
|
||||
echo ""
|
||||
if $DRY_RUN; then
|
||||
echo " This was a DRY RUN. To apply changes:"
|
||||
echo " $0 --apply"
|
||||
fi
|
||||
echo "============================================="
|
||||
|
||||
# Exit with error if any failures
|
||||
[[ "$FAILED" -eq 0 ]] || exit 1
|
||||
Loading…
Reference in a new issue