Audio Transcription And Evaluation With Gemini Flash

pip install -U google-genai "arize-phoenix-evals<3" "arize<8" opentelemetry-api \
  opentelemetry-sdk openinference-semantic-conventions arize-otel

export ARIZE_SPACE_ID="<your-space-id>"
export ARIZE_API_KEY="<your-api-key>"
export GEMINI_API_KEY="<your-gemini-api-key>"
export GOOGLE_CLOUD_PROJECT="<your-google-cloud-project-id>"

curl -sL "https://storage.googleapis.com/generativeai-downloads/data/State_of_the_Union_Address_30_January_1961.mp3" \
  -o sample.mp3

import os

from google import genai

## Audio file url --> allows you to play audio in UI
URL = "https://storage.googleapis.com/generativeai-downloads/data/State_of_the_Union_Address_30_January_1961.mp3"

gemini_client = genai.Client(api_key=os.environ["GEMINI_API_KEY"])

your_file = gemini_client.files.upload(file="sample.mp3")

from opentelemetry import trace
from arize.otel import register
from opentelemetry.trace import Status, StatusCode
from opentelemetry.semconv.trace import SpanAttributes


PROJECT_NAME = "gemini-audio"  # Set this to any name you'd like for your app

# Setup OTel via our convenience function
tracer_provider = register(
    space_id=os.environ["ARIZE_SPACE_ID"],  # in app space settings page
    api_key=os.environ["ARIZE_API_KEY"],  # in app space settings page
    project_name=PROJECT_NAME,
)

trace.set_tracer_provider(tracer_provider)
tracer = trace.get_tracer(__name__)

prompt = "Provide a transcript of the speech from 01:00 to 01:30."

# Call Gemini

with tracer.start_as_current_span(
    "process_audio",
    openinference_span_kind="llm",
) as span:
  span.set_attribute("input.audio.url", URL)
  span.set_attribute("llm.prompts", prompt)
  span.set_attribute("input.value", prompt)
  response = gemini_client.models.generate_content(
    model='gemini-3.5-flash',
    contents=[
      prompt,
      your_file,
    ]
  )
  span.set_attribute("input.audio.transcript", response.text)
  span.set_attribute("output.value", response.text)
  span.set_status(Status(StatusCode.OK))

response.text

pip install "arize[Tracing]>=7.1.0,<8" packaging

# Note: This example uses Python SDK v7
import os
from datetime import datetime, timezone, timedelta

from arize.exporter import ArizeExportClient
from arize.utils.types import Environments

client = ArizeExportClient()

print('#### Exporting your primary dataset into a dataframe.')

primary_df = client.export_model_to_df(
    space_id=os.environ["ARIZE_SPACE_ID"],
    model_id=PROJECT_NAME,
    where="name = 'process_audio'", #Just pull the spans with name = "process_audio"
    environment=Environments.TRACING,
    start_time = datetime.now(timezone.utc) - timedelta(days=1),
    end_time = datetime.now(timezone.utc) #pull traces for the last 24 hours
)

#set the column in the dataframe to match the variable name used in our eval template
primary_df["output"] = primary_df["attributes.output.value"]

SENTIMENT_EVAL_TEMPLATE = """

You are a helpful AI bot that checks for the sentiment in the output text. Your task is to evaluate the sentiment of the given output and categorize it as positive, neutral, or negative.

Here is the data:
[BEGIN DATA]
============
[Output]: {attributes.output.value}
============
[END DATA]

Determine the sentiment of the output based on the content and context provided. Your response should be ONLY a single word, either "positive", "neutral", or "negative", and should not contain any text or characters aside from that word.

Then write out in a step by step manner an EXPLANATION to show how you determined the sentiment of the output.  Do not include any text or characters aside from the EXPLANATION.

Your response should follow the format of the example response below. Provide a single LABEL and a single EXPLANATION. Do not include any special characters in the response. Do not include special characters such as "#" in your response.

Example response:

EXPLANATION: An explanation of your reasoning for why the label is "positive", "neutral", or "negative"
LABEL: "positive" or "neutral" or "negative"

"""

gcloud auth application-default login
gcloud config set project <your-google-cloud-project-id>

#Gemini as LLM as a Judge - LLM Classify
import os

import pandas as pd
from phoenix.evals import (GeminiModel, llm_classify)

#We will use Gemini 2.5 Pro to evaluate the text transcription
project_id = os.environ["GOOGLE_CLOUD_PROJECT"] # Set this to your google project id
gemini_model = GeminiModel(model="gemini-2.5-pro", project=project_id)

rails = ["positive", "neutral", "negative"]

evals_df = llm_classify(
    data=primary_df,
    template=SENTIMENT_EVAL_TEMPLATE,
    model=gemini_model,
    rails=rails,
    provide_explanation=True
)

#set eval labels
evals_df["eval.sentiment.label"] = evals_df["label"]
evals_df["eval.sentiment.explanation"] = evals_df["explanation"]
evals_df["context.span_id"] = primary_df["context.span_id"]

evals_df.head()

# Note: This example uses Python SDK v7
import os

from arize.pandas.logger import Client


# Initialize Arize AX client using the model_id and version you used previously
arize_client = Client(
    space_id=os.environ["ARIZE_SPACE_ID"],
    api_key=os.environ["ARIZE_API_KEY"],
)

# send the evaluation results to Arize AX
arize_client.log_evaluations_sync(evals_df, "gemini-audio")