A strain gate on every prompt change.

name: contradish · CAI strain check

on:
  pull_request:
    paths:
      - 'prompts/**'            # system prompts
      - '**/*prompt*.txt'       # any file with "prompt" in the name
      - '**/*system*.txt'       # system instruction files
      - 'config/**/*.json'      # model config files
      # add any other paths where your prompts or model settings live

jobs:
  strain-check:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install contradish
        run: pip install contradish

      - name: Run CAI benchmark
        id: cai
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          mkdir -p results
          contradish benchmark --model gpt-4o --output-json results/cai.json --judge-votes ${{ vars.CONTRADISH_JUDGE_VOTES || 3 }}
          STRAIN=$(python -c "import json; d=json.load(open('results/cai.json')); print(d['judgment_strain'])")
          CRITICAL=$(python -c "import json; d=json.load(open('results/cai.json')); print(d.get('critical_count', 0))")
          CONFIDENCE=$(python -c "import json; d=json.load(open('results/cai.json')); c=d.get('judge_confidence'); print(c if c is not None else '')")
          VOTES_CAST=$(python -c "import json; d=json.load(open('results/cai.json')); v=d.get('avg_judge_votes_cast'); print(v if v is not None else '')")
          ORDER_SENSITIVE=$(python -c "import json; d=json.load(open('results/cai.json')); print(d.get('judge_order_sensitive_cases', 0))")
          echo "strain=$STRAIN" >> $GITHUB_OUTPUT
          echo "critical=$CRITICAL" >> $GITHUB_OUTPUT
          echo "confidence=$CONFIDENCE" >> $GITHUB_OUTPUT
          echo "votes_cast=$VOTES_CAST" >> $GITHUB_OUTPUT
          echo "order_sensitive=$ORDER_SENSITIVE" >> $GITHUB_OUTPUT

      - name: Post PR comment
        uses: actions/github-script@v7
        with:
          script: |
            const strain = parseFloat('${{ steps.cai.outputs.strain }}');
            const critical = parseInt('${{ steps.cai.outputs.critical }}');
            const confidenceRaw = '${{ steps.cai.outputs.confidence }}';
            const confidence = confidenceRaw ? parseFloat(confidenceRaw) : null;
            const votesCastRaw = '${{ steps.cai.outputs.votes_cast }}';
            const votesCast = votesCastRaw ? parseFloat(votesCastRaw) : null;
            const judgeVoteCap = '${{ vars.CONTRADISH_JUDGE_VOTES || 3 }}';
            const orderSensitive = parseInt('${{ steps.cai.outputs.order_sensitive }}' || '0');
            const threshold = parseFloat('${{ vars.CONTRADISH_THRESHOLD || "0.35" }}');
            const pass = strain <= threshold;
            const emoji = strain <= 0.20 ? '✅' : strain <= threshold ? '⚠️' : '❌';
            const status = strain <= 0.20 ? 'passing' : strain <= threshold ? 'review' : 'failing';
            const lowConfidence = confidence !== null && confidence < 0.7;
            await github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: [
                `## ${emoji} contradish · CAI Strain Check`,
                '',
                '| metric | value |',
                '|--------|-------|',
                `| judgment_strain | \`${strain.toFixed(3)}\` |`,
                `| threshold | \`${threshold}\` |`,
                `| critical failures | \`${critical}\` |`,
                confidence !== null ? `| judge confidence | \`${confidence.toFixed(2)}\` (avg ${votesCast !== null ? votesCast.toFixed(1) : '?'}/${judgeVoteCap} votes cast) |` : null,
                orderSensitive > 0 ? `| order-sensitive cases | \`${orderSensitive}\` |` : null,
                `| status | **${status}** |`,
                '',
                lowConfidence ? `⚠️ **Low judge confidence (${confidence.toFixed(2)}).** The judge disagreed with itself across votes on several cases -- treat this result as provisional and consider re-running before acting on it.` : null,
                lowConfidence ? '' : null,
                orderSensitive > 0 ? `📐 **${orderSensitive} case(s) were order-sensitive** -- the judge's verdict flipped when the same evidence was shown in a different order, not because the content changed. Those specific results are the least trustworthy in this run.` : null,
                orderSensitive > 0 ? '' : null,
                `Run \`contradish findings results/cai.json\` locally for the full diagnosis.`,
                '',
                '_[contradish](https://contradish.com) · memory-aware contradiction detection_'
              ].filter(Boolean).join('\n')
            });

      - name: Enforce strain threshold
        if: steps.cai.outputs.strain > (vars.CONTRADISH_THRESHOLD || '0.35')
        run: |
          echo "❌ Judgment Strain ${{ steps.cai.outputs.strain }} exceeds threshold"
          echo "Run: contradish improve --policy <domain> --model gpt-4o --target-strain 0.20"
          exit 1