mkdir demo
cd demo
spice init

mkdir demo
cd demo
spice init

spice login

spice login

spice connect <username>/evals

spice connect <username>/evals

version: v1beta1
kind: Spicepod
name: demo

dependencies:
  - Jeadie/evals

version: v1beta1
kind: Spicepod
name: demo

dependencies:
  - Jeadie/evals

models:
  - name: gpt-4o
    from: openai:gpt-4o
    params:
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

models:
  - name: gpt-4o
    from: openai:gpt-4o
    params:
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

spice run

spice run

curl -XPOST "http://localhost:8090/v1/evals/taxes"      -H "Content-Type: application/json"      -d '{
    "model": "gpt-4o"
  }' | jq

curl -XPOST "http://localhost:8090/v1/evals/taxes"      -H "Content-Type: application/json"      -d '{
    "model": "gpt-4o"
  }' | jq

spice sql

spice sql

runtime:
  task_history:
    captured_output: truncated

runtime:
  task_history:
    captured_output: truncated

views:
  - name: user_queries
    sql: |
      SELECT
        json_get_json(input, 'messages') AS input,
        json_get_str((captured_output -> 0), 'content') as ideal
      FROM runtime.task_history
      WHERE task='ai_completion'
  - name: latest_eval_runs
    sql: |
      SELECT model, MAX(created_at) as latest_run
         FROM eval.runs
         GROUP BY model
  - name: model_stats
    sql: |
      SELECT
        r.model,
        COUNT(*) as total_queries,
        SUM(CASE WHEN res.value = 1.0 THEN 1 ELSE 0 END) as correct_answers,
        AVG(res.value) as accuracy
      FROM eval.runs r
      JOIN latest_eval_runs lr ON r.model = lr.model AND r.created_at = lr.latest_run
      JOIN eval.results res ON res.run_id = r.id
      GROUP BY r.model

evals:
  - name: mimic-user-queries
    description: |
      Evaluates how well a model can copy the exact answers already returned to a user. Useful for testing if a smaller/cheaper model is sufficient.
    dataset: user_queries
    scorers:
      - match

views:
  - name: user_queries
    sql: |
      SELECT
        json_get_json(input, 'messages') AS input,
        json_get_str((captured_output -> 0), 'content') as ideal
      FROM runtime.task_history
      WHERE task='ai_completion'
  - name: latest_eval_runs
    sql: |
      SELECT model, MAX(created_at) as latest_run
         FROM eval.runs
         GROUP BY model
  - name: model_stats
    sql: |
      SELECT
        r.model,
        COUNT(*) as total_queries,
        SUM(CASE WHEN res.value = 1.0 THEN 1 ELSE 0 END) as correct_answers,
        AVG(res.value) as accuracy
      FROM eval.runs r
      JOIN latest_eval_runs lr ON r.model = lr.model AND r.created_at = lr.latest_run
      JOIN eval.results res ON res.run_id = r.id
      GROUP BY r.model

evals:
  - name: mimic-user-queries
    description: |
      Evaluates how well a model can copy the exact answers already returned to a user. Useful for testing if a smaller/cheaper model is sufficient.
    dataset: user_queries
    scorers:
      - match

models:
  - name: llama3
    from: huggingface:huggingface.co/meta-llama/Llama-3.2-3B-Instruct
    params:
      hf_token: ${ secrets:SPICE_HUGGINGFACE_API_KEY }

  - name: gpt-4o # Keep previous model.

models:
  - name: llama3
    from: huggingface:huggingface.co/meta-llama/Llama-3.2-3B-Instruct
    params:
      hf_token: ${ secrets:SPICE_HUGGINGFACE_API_KEY }

  - name: gpt-4o # Keep previous model.

spice models

spice models

NAME    FROM                                                         STATUS
gpt-4o  openai:gpt-4o                                                ready
llama3  huggingface:huggingface.co/meta-llama/Llama-3.3-70B-Instruct ready

NAME    FROM                                                         STATUS
gpt-4o  openai:gpt-4o                                                ready
llama3  huggingface:huggingface.co/meta-llama/Llama-3.3-70B-Instruct ready

spice run

spice run

spice chat

spice chat

# Run eval with GPT-4
curl -XPOST "http://localhost:8090/v1/evals/mimic-user-queries" \
  -H "Content-Type: application/json" \
  -d '{"model": "gpt-4o"}' | jq

# Run eval with LLaMA
curl -XPOST "http://localhost:8090/v1/evals/mimic-user-queries" \
  -H "Content-Type: application/json" \
  -d '{"model": "llama3"}' | jq

# Run eval with GPT-4
curl -XPOST "http://localhost:8090/v1/evals/mimic-user-queries" \
  -H "Content-Type: application/json" \
  -d '{"model": "gpt-4o"}' | jq

# Run eval with LLaMA
curl -XPOST "http://localhost:8090/v1/evals/mimic-user-queries" \
  -H "Content-Type: application/json" \
  -d '{"model": "llama3"}' | jq

spice sql

spice sql

version: v1beta1
kind: Spicepod
name: demo

dependencies:
  - Jeadie/evals

runtime:
  task_history:
    captured_output: truncated

views:
  - name: user_queries
    sql: |
      SELECT
        json_get_json(input, 'messages') AS input,
        json_get_str((captured_output -> 0), 'content') as ideal
      FROM runtime.task_history
      WHERE task='ai_completion'

evals:
  - name: mimic-user-queries
    description: |
      Evaluates how well a model can copy the exact answers already returned to a user. Useful for testing if a smaller/cheaper model is sufficient.
    dataset: user_queries
    scorers:
      - match

models:
  - name: gpt-4o
    from: openai:gpt-4o
    params:
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

  - name: llama3
    from: huggingface:huggingface.co/meta-llama/Llama-3.2-3B-Instruct
    params:
      hf_token: ${ secrets:SPICE_HUGGINGFACE_API_KEY }

version: v1beta1
kind: Spicepod
name: demo

dependencies:
  - Jeadie/evals

runtime:
  task_history:
    captured_output: truncated

views:
  - name: user_queries
    sql: |
      SELECT
        json_get_json(input, 'messages') AS input,
        json_get_str((captured_output -> 0), 'content') as ideal
      FROM runtime.task_history
      WHERE task='ai_completion'

evals:
  - name: mimic-user-queries
    description: |
      Evaluates how well a model can copy the exact answers already returned to a user. Useful for testing if a smaller/cheaper model is sufficient.
    dataset: user_queries
    scorers:
      - match

models:
  - name: gpt-4o
    from: openai:gpt-4o
    params:
      openai_api_key: ${ secrets:SPICE_OPENAI_API_KEY }

  - name: llama3
    from: huggingface:huggingface.co/meta-llama/Llama-3.2-3B-Instruct
    params:
      hf_token: ${ secrets:SPICE_HUGGINGFACE_API_KEY }

SELECT
  input,
  output,
  actual
FROM eval.results
WHERE value=0.0 LIMIT 5;

SELECT
  input,
  output,
  actual
FROM eval.results
WHERE value=0.0 LIMIT 5;

SELECT
  model,
  total_queries,
  correct_answers,
  ROUND(accuracy * 100, 2) as accuracy_percentage
FROM model_stats
ORDER BY accuracy_percentage DESC;

SELECT
  model,
  total_queries,
  correct_answers,
  ROUND(accuracy * 100, 2) as accuracy_percentage
FROM model_stats
ORDER BY accuracy_percentage DESC;