1name: Run new evals
2
3on:
4  workflow_dispatch:
5  pull_request:
6    branches:
7      - main
8    paths:
9      - 'evals/registry/**'
10
11jobs:
12  check_files:
13    runs-on: ubuntu-latest
14
15    steps:
16    - name: Checkout repository
17      uses: actions/checkout@v2
18      with:
19        fetch-depth: 0
20        lfs: true
21
22    - name: Install Git LFS
23      run: |
24        sudo apt-get install git-lfs
25        git lfs install
26
27    - name: Set up Python
28      uses: actions/setup-python@v2
29      with:
30        python-version: 3.9
31
32    - name: Install dependencies
33      run: |
34        python -m pip install --upgrade pip
35        pip install pyyaml
36        pip install -e .
37
38    - name: Get list of new YAML files in evals/registry/evals
39      id: get_files
40      run: |
41        # Use environment files to store the output
42        git diff --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^evals/registry/evals/.*\.yaml$' | xargs > new_files
43        echo "new_files=$(cat new_files)" >> $GITHUB_ENV
44
45    - name: Run oaieval command for each new YAML file
46      env:
47        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
48      run: |
49        files="${{ env.new_files }}"
50        if [ -n "$files" ]; then
51          for file in $files; do
52            echo "Processing $file"
53            first_key=$(python .github/workflows/parse_yaml.py $file)
54            echo "Eval Name: $first_key"
55            oaieval dummy $first_key --max_samples 10
56          done
57        else
58          echo "No new YAML files found in evals/registry/evals"
59        fi
60

1name: Run new evals
2
3on:
4  workflow_dispatch:
5  pull_request:
6    branches:
7      - main
8    paths:
9      - 'evals/registry/**'
10
11jobs:
12  check_files:
13    runs-on: ubuntu-latest
14
15    steps:
16    - name: Checkout repository
17      uses: actions/checkout@v2
18      with:
19        fetch-depth: 0
20        lfs: true
21
22    - name: Install Git LFS
23      run: |
24        sudo apt-get install git-lfs
25        git lfs install
26
27    - name: Set up Python
28      uses: actions/setup-python@v2
29      with:
30        python-version: 3.9
31
32    - name: Install dependencies
33      run: |
34        python -m pip install --upgrade pip
35        pip install pyyaml
36        pip install -e .
37
38    - name: Get list of new YAML files in evals/registry/evals
39      id: get_files
40      run: |
41        # Use environment files to store the output
42        git diff --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^evals/registry/evals/.*\.yaml$' | xargs > new_files
43        echo "new_files=$(cat new_files)" >> $GITHUB_ENV
44
45    - name: Run oaieval command for each new YAML file
46      env:
47        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
48      run: |
49        files="${{ env.new_files }}"
50        if [ -n "$files" ]; then
51          for file in $files; do
52            echo "Processing $file"
53            first_key=$(python .github/workflows/parse_yaml.py $file)
54            echo "Eval Name: $first_key"
55            oaieval dummy $first_key --max_samples 10
56          done
57        else
58          echo "No new YAML files found in evals/registry/evals"
59        fi
60