Add LLM checking workflow again. (#6038)

.github/workflows/get_real_pr_shas.yml

@@ -29,6 +29,7 @@
 #
 # Use `${{ needs.real_pr_shas.outputs.BASE_SHA }}` to get the real base sha.
 # Use `${{ needs.real_pr_shas.outputs.HEAD_SHA }}` to get the PR head commit sha.
+# Use `${{ needs.real_pr_shas.outputs.COMMIT_MSGS }}` to get the commit messages of git log --pretty=%B BASE_SHA..HEAD_SHA
 #
 # This workflow is copied from https://github.com/JensDll/should-run/blob/main/.github/workflows/main.yaml#L54
 #
@@ -47,6 +48,9 @@ on:
       HEAD_SHA:
         description: "The real PR commit HEAD sha"
         value: ${{ jobs.get_real_pr_shas.outputs.output_head }}
+      COMMIT_MSGS:
+        description: "The commit messages of BASE_SHA..HEAD_SHA"
+        value: ${{ jobs.get_real_pr_shas.outputs.output_msgs }}
 
 jobs:
   get_real_pr_shas:
@@ -54,6 +58,7 @@ jobs:
     outputs:
       output_base: ${{ steps.get_shas.outputs.BASE_SHA }}
       output_head: ${{ steps.get_shas.outputs.HEAD_SHA }}
+      output_msgs: ${{ steps.get_shas.outputs.COMMIT_MSGS }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -68,10 +73,18 @@ jobs:
             git fetch --no-tags --prune --no-recurse-submodules --deepen=10 origin ${{ github.event.pull_request.base.sha }}
           done
 
-          base=$(git rev-list ${{ github.event.pull_request.head.sha }} ^${{ github.event.pull_request.base.sha }} | tail --lines 1 | xargs -I {} git rev-parse {}~1)
+          BASE_SHA=$(git rev-list ${{ github.event.pull_request.head.sha }} ^${{ github.event.pull_request.base.sha }} | tail --lines 1 | xargs -I {} git rev-parse {}~1)
+          HEAD_SHA=${{ github.event.pull_request.head.sha }}
+          COMMIT_MSGS=$(git log --pretty=%B "$BASE_SHA".."$HEAD_SHA")
+          if [[ $? -eq 128 ]]; then
+          	echo "Failed to get log"
+          	exit 2
+          fi
 
-          echo "BASE_SHA=$base"
-          echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}"
+          echo "BASE_SHA=$BASE_SHA"
+          echo "HEAD_SHA=$HEAD_SHA"
+          echo -e "COMMIT_MSGS:\n$COMMIT_MSGS"
 
-          echo "BASE_SHA=$base" >> $GITHUB_OUTPUT
-          echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
+          echo "BASE_SHA=$BASE_SHA" >> $GITHUB_OUTPUT
+          echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_OUTPUT
+          echo "COMMIT_MSGS=$COMMIT_MSGS" >> $GITHUB_OUTPUT

.github/workflows/llm_checks.yml

@@ -0,0 +1,77 @@
+# SECURITY:
+#
+# This workflow runs on pull_request_target and has write privileges.
+# Those are used to add labels to the PR.
+#
+# **IF a user can run code in here, they would be able to extract our secrets.**
+#
+# This is why it doesn't run external scripts.
+# The exception is the get_real_pr_shas.yml workflow.
+# The input of it is santized.
+
+name: AI/LLM Checks
+
+on:
+  pull_request_target:
+
+jobs:
+  real_pr_shas:
+    uses: rizinorg/rizin/.github/workflows/get_real_pr_shas.yml@dev
+
+  ai_checks:
+    name: LLM checks
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    needs: real_pr_shas
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0  # Full history needed for commit scanning
+
+      - name: Validate input hashes
+        env:
+          BASE_SHA: ${{ needs.real_pr_shas.outputs.BASE_SHA }}
+          HEAD_SHA: ${{ needs.real_pr_shas.outputs.HEAD_SHA }}
+        run: |
+          if echo "$BASE_SHA" | grep -q "[^a-f0-9]"; then
+            echo "BASE_SHA is malformed: $BASE_SHA"
+            exit 1
+          fi
+          if echo "$HEAD_SHA" | grep -q "[^a-f0-9]"; then
+            echo "HEAD_SHA is malformed: $HEAD_SHA"
+            exit 1
+          fi
+
+      - name: Check AGENT.md is unchanged
+        env:
+          BASE_SHA: ${{ needs.real_pr_shas.outputs.BASE_SHA }}
+        run: |
+          diff=$(git diff --name-status "$BASE_SHA" AGENTS.md)
+          if [[ $? -eq 128 ]]; then
+          	echo "Failed to diff"
+          	exit 1
+          fi
+
+          if [[ -n "$diff" ]]; then
+          	echo "Edits to 'AGENT.md' are not allowed!"
+          	exit 1
+          fi
+
+      - name: Check for AI usage and label
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.number }}
+          BASE_SHA: ${{ needs.real_pr_shas.outputs.BASE_SHA }}
+          HEAD_SHA: ${{ needs.real_pr_shas.outputs.HEAD_SHA }}
+          COMMIT_MSGS: ${{ needs.real_pr_shas.outputs.COMMIT_MSGS }}
+        run: |
+          LABEL_NAME="AI/LLM"
+          NEEDLE="Co-authored-by agent"
+
+          if echo "$COMMIT_MSGS" | grep -q "$NEEDLE"; then
+            echo "Authored by AI agent"
+            gh pr --repo $REPO edit $NUMBER --add-label "$LABEL_NAME"
+          fi