Relevance Classification Evaluation

df = download_benchmark_dataset(
    task="binary-relevance-classification",
    dataset_name="wiki_qa-train"
)

N_EVAL_SAMPLE_SIZE = 100
df_sample = df.sample(n=N_EVAL_SAMPLE_SIZE).reset_index(drop=True)
df_sample = df_sample.rename(columns={
    "query_text": "input",
    "document_text": "reference",
})

from phoenix.evals import LLM, async_evaluate_dataframe
from phoenix.evals.metrics import DocumentRelevanceEvaluator

llm = LLM(provider="openai", model="gpt-4")
relevance_evaluator = DocumentRelevanceEvaluator(llm=llm)

evals_df = await async_evaluate_dataframe(dataframe=df_sample, evaluators=[relevance_evaluator], concurrency=10)
relevance_classifications = evals_df["document_relevance_score"].str["label"].tolist()
choices = relevance_evaluator.CHOICES

true_labels = df_sample["relevant"].map({True: "relevant", False: "unrelated"}).tolist()

print(classification_report(true_labels, relevance_classifications, labels=choices))
confusion_matrix = ConfusionMatrix(
    actual_vector=true_labels, predict_vector=relevance_classifications, classes=choices
)
confusion_matrix.plot(
    cmap=plt.colormaps["Blues"],
    number_label=True,
    normalized=True,
)

relevance_classifications_df = await async_evaluate_dataframe(
    dataframe=df_sample.sample(n=5),
    evaluators=[relevance_evaluator],
    concurrency=10,
)
relevance_classifications_df["label"] = relevance_classifications_df["document_relevance_score"].str[
    "label"
]
relevance_classifications_df["explanation"] = relevance_classifications_df[
    "document_relevance_score"
].str["explanation"]

# GPT-3.5
llm_gpt35 = LLM(provider="openai", model="gpt-3.5-turbo")

# GPT-4 Turbo
llm_gpt4turbo = LLM(provider="openai", model="gpt-4-turbo-preview")