Using CI/CD

name: test-trusys-cli

on:
  push:
    tags:
      - 'test-cli*'

jobs:
  test-trusys-cli:
    runs-on: ubuntu-latest
    env:
      TRUSYS_APPLICATION: GPT4.1  # Specify your application here
      TRUSYS_LIBRARY: Test2       # Specify your prompt library name here
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install trusys CLI
        run: npm install -g trusys

      - name: Run trusys  # Add the Trusys API key in Github Secrets as TRUSYS_API_KEY
        run: |
          npx trusys auth login --api-key ${{ secrets.TRUSYS_API_KEY }}
          npx trusys eval --run --application "${{ env.TRUSYS_APPLICATION }}" --library "${{ env.TRUSYS_LIBRARY }}"

name: metric-test-trusys-cli

on:
  push:
    tags:
      - 'metric-test-cli*'

jobs:
  test-trusys-cli:
    runs-on: ubuntu-latest
    env:
      TRUSYS_APPLICATION: GPT4.1                              # Specify your application here
      TRUSYS_LIBRARY: Test2                                   # Specify your prompt library name here
      TRUSYS_RELEVANT_ASSERTION_TYPES: icontains,llm-rubric   # Comma-separated list of relevant assertion types to validate
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install trusys CLI
        run: npm install -g trusys

      - name: Run trusys          # Add the Trusys API key in Github Secrets as TRUSYS_API_KEY
        continue-on-error: true   # Allow this step to continue even if failed to capture results for validation
        run: |
          npx trusys auth login --api-key ${{ secrets.TRUSYS_API_KEY }}
          npx trusys eval --run --application "${{ env.TRUSYS_APPLICATION }}" --library "${{ env.TRUSYS_LIBRARY }}" --output "results.json"


      - name: Validate trusys results
        run: |
          set -euo pipefail
          node - <<'NODE'
          const fs = require('fs');
          const path = 'results.json';

          if (!fs.existsSync(path)) {
            console.error(`Results file not found: ${path}`);
            process.exit(1);
          }

          let parsed;
          try {
            parsed = JSON.parse(fs.readFileSync(path, 'utf8'));
          } catch (err) {
            console.error(`Failed to parse JSON from ${path}:`, err);
            process.exit(1);
          }

          const body = (((parsed.results || {}).table || {}).body) || [];
          const annotatedResults = body.flatMap(row => {
            const prompt = row?.prompt ?? '';
            return (row.outputs || []).flatMap(output => {
              const componentResults = ((output.gradingResult || {}).componentResults) || [];
              return componentResults.map(result => ({ result, prompt }));
            });
          });

          const relevantTypes = new Set(
            (process.env.TRUSYS_RELEVANT_ASSERTION_TYPES || '')
              .split(',')
              .map(type => type.trim().toLowerCase())
              .filter(type => type.length > 0)
          );
          const relevant = annotatedResults.filter(({ result }) => {
            const assertionType = result?.assertion?.type;
            if (typeof assertionType !== 'string') {
              return false;
            }
            const normalizedType = assertionType.trim().toLowerCase();
            return relevantTypes.has(normalizedType);
          });

          const failing = relevant.filter(({ result }) => result?.pass === false);

          if (failing.length > 0) {
            console.error(`Found ${failing.length} failing assertion(s) in ${path}`);
            failing.forEach(({ result, prompt }, index) => {
              const type = result?.assertion?.type ?? 'unknown';
              const reason = result?.reason ?? 'No reason provided';
              const promptSnippet = prompt;
              console.error(`  [${index + 1}] prompt=${JSON.stringify(promptSnippet)} :: assertion.type=${type} :: ${reason}`);
            });
            process.exit(1);
          }

          console.log(`All relevant assertions (${relevant.length}) passed in ${path}`);
          NODE