Session-Level Evals

import pandas as pd
def prepare_sessions(df: pd.DataFrame) -> pd.DataFrame:
    """Collapse spans into a single row per session with ordered user/assistant messages."""
    sessions = []
    # Sort spans chronologically, then group by the session identifier
    grouped = (
        df.sort_values("start_time")
        .groupby("attributes.session.id", as_index=False)
    )

    for session_id, group in grouped:
        sessions.append(
            {
                "session_id": session_id,
                # Collect all user inputs for the session (dropping any nulls)
                "user_inputs": group["attributes.input.value"].dropna().tolist(),
                # Collect all assistant responses for the session (dropping any nulls)
                "output_messages": group["attributes.output.value"].dropna().tolist(),
                # Count how many distinct traces are in this session
                "trace_count": group["context.trace_id"].nunique(),
            }
        )
    return pd.DataFrame(sessions)

# Build the session-level dataframe using the filtered traces from step 2
sessions_df = prepare_sessions(primary_df)

SESSION_CORRECTNESS_PROMPT = """
You are a helpful AI bot that evaluates the effectiveness and correctness of an AI agent's session.

A session consists of multiple traces (interactions) between a user and an AI system. I will provide you with:
1. The user inputs that initiated each trace in the session, in chronological order.
2. The AI's output messages for each trace in the session, in chronological order.

An effective and correct session:
- Shows consistent understanding of user intentions across traces
- Maintains context and coherence between interactions
- Successfully achieves the overall user goals
- Builds upon previous interactions in the conversation

##
User Inputs:
{user_inputs}

Output Messages:
{output_messages}
##

Evaluate the session based on the given criteria. Your response must be a single string, either `correct` or `incorrect`, and must not include any additional text.

- Respond with `correct` if the session effectively accomplishes user goals with appropriate responses and coherence.
- Respond with `incorrect` if the session shows confusion, inappropriate responses, or fails to accomplish user goals.
"""

from phoenix.evals import create_classifier
from phoenix.evals.evaluators import async_evaluate_dataframe
from phoenix.evals.llm import LLM

llm = LLM(provider="openai", model="gpt-4o-mini")

correctness_evaluator = create_classifier(
    name="correctness",
    llm=llm,
    prompt_template=SESSION_CORRECTNESS_PROMPT,
    choices={"correct": 1.0, "incorrect": 0.0},
)

results_df = await async_evaluate_dataframe(
    dataframe=sessions_df,
    evaluators=[correctness_evaluator],
)