{
    budget: "low",
    location: "San Francisco",
    purpose: "pleasure"
}

parameters_schema = {
    "type": "object",
    "properties": {
        "location": {
            "type": "string",
            "description": 'The desired destination location. Use city, state, and country format when possible. If no destination is provided, return "unstated".',
        },
        "budget_level": {
            "type": "string",
            "enum": ["low", "medium", "high", "not_stated"],
            "description": 'The desired budget level. If no budget level is provided, return "not_stated".',
        },
        "purpose": {
            "type": "string",
            "enum": ["business", "pleasure", "other", "non_stated"],
            "description": 'The purpose of the trip. If no purpose is provided, return "not_stated".',
        },
    },
    "required": ["location", "budget_level", "purpose"],
}
function_schema = {
    "name": "record_travel_request_attributes",
    "description": "Records the attributes of a travel request",
    "parameters": parameters_schema,
}
system_message = (
    "You are an assistant that parses and records the attributes of a user's travel request."
)

response = openai.ChatCompletion.create(
    model=model,
    messages=[
        {"role": "system", "content": system_message},
        {"role": "user", "content": travel_request},
    ],
    functions=[function_schema],
    # By default, the LLM will choose whether or not to call a function given the conversation context.
    # The line below forces the LLM to call the function so that the output conforms to the schema.
    function_call={"name": function_schema["name"]},
)

!pip install -qqq "arize-phoenix>=8.0.0" datasets openinference-instrumentation-openai

import os
from getpass import getpass

os.environ["PHOENIX_COLLECTOR_ENDPOINT"] = "https://app.phoenix.arize.com"
if not os.environ.get("PHOENIX_CLIENT_HEADERS"):
    os.environ["PHOENIX_CLIENT_HEADERS"] = "api_key=" + getpass("Enter your Phoenix API key: ")

if not os.environ.get("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = getpass("Enter your OpenAI API key: ")

from datasets import load_dataset

ds = load_dataset("syeddula/fridgeReviews")["train"]
ds = ds.to_pandas()
ds.head()

import uuid

import phoenix as px
from phoenix.client import Client as PhoenixClient

unique_id = uuid.uuid4()

# Upload the dataset to Phoenix
dataset = px.Client().upload_dataset(
    dataframe=ds,
    input_keys=["Review"],
    output_keys=["Sentiment"],
    dataset_name=f"review-classification-{unique_id}",
)

from openai import OpenAI
from openai.types.chat.completion_create_params import CompletionCreateParamsBase

from phoenix.client.types import PromptVersion

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {
            "role": "system",
            "content": "You are an evaluator who assesses the sentiment of a review. Output if the review positive, negative, or neutral. Only respond with one of these classifications.",
        },
        {"role": "user", "content": "{{Review}}"},
    ],
)

prompt_identifier = "fridge-sentiment-reviews"

prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="A prompt for classifying reviews based on sentiment.",
    version=PromptVersion.from_openai(params),
)

def zero_shot_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(**prompt.format(variables={"Review": input["Review"]}))
    return resp.choices[0].message.content.strip()


def evaluate_response(output, expected):
    return output.lower() == expected["Sentiment"].lower()

from openinference.instrumentation.openai import OpenAIInstrumentor

from phoenix.otel import register

tracer_provider = register(project_name="few-shot-examples")
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

import nest_asyncio

from phoenix.experiments import run_experiment

nest_asyncio.apply()

initial_experiment = run_experiment(
    dataset,
    task=zero_shot_prompt,
    evaluators=[evaluate_response],
    experiment_description="Zero-Shot Prompt",
    experiment_name="zero-shot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + prompt.id},
)

ds = load_dataset("syeddula/fridgeReviews")["test"]
one_shot_example = ds.to_pandas().sample(1)

one_shot_template = """
"You are an evaluator who assesses the sentiment of a review. Output if the review positive, negative, or neutral. Only respond with one of these classifications."

Here is one example of a review and the sentiment:

{examples}
"""

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": one_shot_template.format(examples=one_shot_example)},
        {"role": "user", "content": "{{Review}}"},
    ],
)

one_shot_prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="One-shot prompt for classifying reviews based on sentiment.",
    version=PromptVersion.from_openai(params),
)

def one_shot_prompt_template(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **one_shot_prompt.format(variables={"Review": input["Review"]})
    )
    return resp.choices[0].message.content.strip()

one_shot_experiment = run_experiment(
    dataset,
    task=one_shot_prompt_template,
    evaluators=[evaluate_response],
    experiment_description="One-Shot Prompting",
    experiment_name="one-shot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + one_shot_prompt.id},
)

ds = load_dataset("syeddula/fridgeReviews")["test"]
few_shot_examples = ds.to_pandas().sample(10)

few_shot_template = """
"You are an evaluator who assesses the sentiment of a review. Output if the review positive, negative, or neutral. Only respond with one of these classifications."

Here are examples of a review and the sentiment:

{examples}
"""
params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": few_shot_template.format(examples=few_shot_examples)},
        {"role": "user", "content": "{{Review}}"},
    ],
)

few_shot_prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Few-shot prompt for classifying reviews based on sentiment.",
    version=PromptVersion.from_openai(params),
)

def few_shot_prompt_template(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **few_shot_prompt.format(variables={"Review": input["Review"]})
    )
    return resp.choices[0].message.content.strip()

few_shot_experiment = run_experiment(
    dataset,
    task=few_shot_prompt_template,
    evaluators=[evaluate_response],
    experiment_description="Few Shot Prompting",
    experiment_name="few-shot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + few_shot_prompt.id},
)

pip install anthropic "arize-phoenix>=4.6.0" openai openinference-instrumentation-openai rouge tiktoken

from typing import Any, Dict

import nest_asyncio
import pandas as pd

nest_asyncio.apply()  # needed for concurrent evals in notebook environments
pd.set_option("display.max_colwidth", None)  # display full cells of dataframes

import phoenix as px

px.launch_app()

from openinference.instrumentation.openai import OpenAIInstrumentor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk import trace as trace_sdk
from opentelemetry.sdk.trace.export import SimpleSpanProcessor

endpoint = "http://127.0.0.1:6006/v1/traces"
tracer_provider = trace_sdk.TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint)))

OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

from datetime import datetime

from datasets import load_dataset

hf_ds = load_dataset("abisee/cnn_dailymail", "3.0.0")
df = (
    hf_ds["test"]
    .to_pandas()
    .sample(n=10, random_state=0)
    .set_index("id")
    .rename(columns={"highlights": "summary"})
)
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
dataset = px.Client().upload_dataset(
    dataframe=df,
    input_keys=["article"],
    output_keys=["summary"],
    dataset_name=f"news-article-summaries-{now}",
)

import os
from getpass import getpass

if os.environ.get("OPENAI_API_KEY") is None:
    os.environ["OPENAI_API_KEY"] = getpass("🔑 Enter your OpenAI API key: ")

from openai import AsyncOpenAI
from phoenix.experiments import Example

openai_client = AsyncOpenAI()


async def summarize_article_openai(example: Example, prompt_template: str, model: str) -> str:
    formatted_prompt_template = prompt_template.format(article=example.input["article"])
    response = await openai_client.chat.completions.create(
        model=model,
        messages=[
            {"role": "assistant", "content": formatted_prompt_template},
        ],
    )
    assert response.choices
    return response.choices[0].message.content

import textwrap
from functools import partial

template = """
Summarize the article in two to four sentences:

ARTICLE
=======
{article}

SUMMARY
=======
"""
gpt_4o = "gpt-4o-2024-05-13"
task = partial(summarize_article_openai, prompt_template=template, model=gpt_4o)
test_example = dataset.examples[0]
print(textwrap.fill(await task(test_example), width=100))

import tiktoken
from rouge import Rouge


# convenience functions
def _rouge_1(hypothesis: str, reference: str) -> Dict[str, Any]:
    scores = Rouge().get_scores(hypothesis, reference)
    return scores[0]["rouge-1"]


def _rouge_1_f1_score(hypothesis: str, reference: str) -> float:
    return _rouge_1(hypothesis, reference)["f"]


def _rouge_1_precision(hypothesis: str, reference: str) -> float:
    return _rouge_1(hypothesis, reference)["p"]


def _rouge_1_recall(hypothesis: str, reference: str) -> float:
    return _rouge_1(hypothesis, reference)["r"]


# evaluators
def rouge_1_f1_score(output: str, expected: Dict[str, Any]) -> float:
    return _rouge_1_f1_score(hypothesis=output, reference=expected["summary"])


def rouge_1_precision(output: str, expected: Dict[str, Any]) -> float:
    return _rouge_1_precision(hypothesis=output, reference=expected["summary"])


def rouge_1_recall(output: str, expected: Dict[str, Any]) -> float:
    return _rouge_1_recall(hypothesis=output, reference=expected["summary"])


def num_tokens(output: str) -> int:
    encoding = tiktoken.encoding_for_model(gpt_4o)
    return len(encoding.encode(output))


EVALUATORS = [rouge_1_f1_score, rouge_1_precision, rouge_1_recall, num_tokens]

from phoenix.experiments import run_experiment

experiment_results = run_experiment(
    dataset,
    task,
    experiment_name="initial-template",
    experiment_description="first experiment using a simple prompt template",
    experiment_metadata={"vendor": "openai", "model": gpt_4o},
    evaluators=EVALUATORS,
)

template = """
Summarize the article in two to four sentences. Be concise and include only the most important information.

ARTICLE
=======
{article}

SUMMARY
=======
"""
task = partial(summarize_article_openai, prompt_template=template, model=gpt_4o)
experiment_results = run_experiment(
    dataset,
    task,
    experiment_name="concise-template",
    experiment_description="explicitly instuct the llm to be concise",
    experiment_metadata={"vendor": "openai", "model": gpt_4o},
    evaluators=EVALUATORS,
)

# examples to include (not included in the uploaded dataset)
train_df = (
    hf_ds["train"]
    .to_pandas()
    .sample(n=5, random_state=42)
    .head()
    .rename(columns={"highlights": "summary"})
)

example_template = """
ARTICLE
=======
{article}

SUMMARY
=======
{summary}
"""

examples = "\n".join(
    [
        example_template.format(article=row["article"], summary=row["summary"])
        for _, row in train_df.iterrows()
    ]
)

template = """
Summarize the article in two to four sentences. Be concise and include only the most important information, as in the examples below.

EXAMPLES
========

{examples}


Now summarize the following article.

ARTICLE
=======
{article}

SUMMARY
=======
"""

template = template.format(
    examples=examples,
    article="{article}",
)
print(template)

task = partial(summarize_article_openai, prompt_template=template, model=gpt_4o)
experiment_results = run_experiment(
    dataset,
    task,
    experiment_name="few-shot-template",
    experiment_description="include examples",
    experiment_metadata={"vendor": "openai", "model": gpt_4o},
    evaluators=EVALUATORS,
)

import os
from getpass import getpass

if os.environ.get("ANTHROPIC_API_KEY") is None:
    os.environ["ANTHROPIC_API_KEY"] = getpass("🔑 Enter your Anthropic API key: ")

from anthropic import AsyncAnthropic

client = AsyncAnthropic()


async def summarize_article_anthropic(example: Example, prompt_template: str, model: str) -> str:
    formatted_prompt_template = prompt_template.format(article=example.input["article"])
    message = await client.messages.create(
        model=model,
        max_tokens=1024,
        messages=[{"role": "user", "content": formatted_prompt_template}],
    )
    return message.content[0].text


claude_35_sonnet = "claude-3-5-sonnet-20240620"
task = partial(summarize_article_anthropic, prompt_template=template, model=claude_35_sonnet)

experiment_results = run_experiment(
    dataset,
    task,
    experiment_name="anthropic-few-shot",
    experiment_description="anthropic",
    experiment_metadata={"vendor": "anthropic", "model": claude_35_sonnet},
    evaluators=EVALUATORS,
)

Here is a concise 3-sentence summary of the article...

pip install 'arize-phoenix>=4.6.0' openai duckdb datasets pyarrow pydantic nest_asyncio openinference-instrumentation-openai --quiet

import phoenix as px

px.launch_app()

from openinference.instrumentation.openai import OpenAIInstrumentor

from phoenix.otel import register

tracer_provider = register()
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

import nest_asyncio

nest_asyncio.apply()

import os
from getpass import getpass

if not os.getenv("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = getpass("🔑 Enter your OpenAI API key: ")

import duckdb
from datasets import load_dataset

data = load_dataset("suzyanil/nba-data")["train"]

conn = duckdb.connect(database=":memory:", read_only=False)
conn.register("nba", data.to_pandas())

conn.query("SELECT * FROM nba LIMIT 5").to_df().to_dict(orient="records")[0]

import os

import openai

client = openai.AsyncClient()

columns = conn.query("DESCRIBE nba").to_df().to_dict(orient="records")

# We will use GPT4o to start
TASK_MODEL = "gpt-4o"
CONFIG = {"model": TASK_MODEL}


system_prompt = (
    "You are a SQL expert, and you are given a single table named nba with the following columns:\n"
    f"{",".join(column["column_name"] + ": " + column["column_type"] for column in columns)}\n"
    "Write a SQL query corresponding to the user's request. Return just the query text, "
    "with no formatting (backticks, markdown, etc.)."
)


async def generate_query(input):
    response = await client.chat.completions.create(
        model=TASK_MODEL,
        temperature=0,
        messages=[
            {
                "role": "system",
                "content": system_prompt,
            },
            {
                "role": "user",
                "content": input,
            },
        ],
    )
    return response.choices[0].message.content

query = await generate_query("Who won the most games?")
print(query)

def execute_query(query):
    return conn.query(query).fetchdf().to_dict(orient="records")


execute_query(query)

questions = [
    "Which team won the most games?",
    "Which team won the most games in 2015?",
    "Who led the league in 3 point shots?",
    "Which team had the biggest difference in records across two consecutive years?",
    "What is the average number of free throws per year?",
]

import pandas as pd

ds = px.Client().upload_dataset(
    dataset_name="nba-questions",
    dataframe=pd.DataFrame([{"question": question} for question in questions]),
    input_keys=["question"],
    output_keys=[],
)

# If you have already uploaded the dataset, you can fetch it using the following line
# ds = px.Client().get_dataset(name="nba-questions")

async def text2sql(question):
    query = await generate_query(question)
    results = None
    error = None
    try:
        results = execute_query(query)
    except duckdb.Error as e:
        error = str(e)

    return {
        "query": query,
        "results": results,
        "error": error,
    }

# Test if there are no sql execution errors
def no_error(output):
    return 1.0 if output.get("error") is None else 0.0


# Test if the query has results
def has_results(output):
    results = output.get("results")
    has_results = results is not None and len(results) > 0
    return 1.0 if has_results else 0.0

import phoenix as px
from phoenix.experiments import run_experiment


# Define the task to run text2sql on the input question
def task(input):
    return text2sql(input["question"])


experiment = run_experiment(
    ds, task=task, evaluators=[no_error, has_results], experiment_metadata=CONFIG
)

samples = conn.query("SELECT * FROM nba LIMIT 1").to_df().to_dict(orient="records")[0]
sample_rows = "\n".join(
    f"{column['column_name']} | {column['column_type']} | {samples[column['column_name']]}"
    for column in columns
)
system_prompt = (
    "You are a SQL expert, and you are given a single table named nba with the following columns:\n\n"
    "Column | Type | Example\n"
    "-------|------|--------\n"
    f"{sample_rows}\n"
    "\n"
    "Write a DuckDB SQL query corresponding to the user's request. "
    "Return just the query text, with no formatting (backticks, markdown, etc.)."
)


async def generate_query(input):
    response = await client.chat.completions.create(
        model=TASK_MODEL,
        temperature=0,
        messages=[
            {
                "role": "system",
                "content": system_prompt,
            },
            {
                "role": "user",
                "content": input,
            },
        ],
    )
    return response.choices[0].message.content


print(await generate_query("Which team won the most games in 2015?"))

experiment = run_experiment(
    ds, task=task, evaluators=[has_results, no_error], experiment_metadata=CONFIG
)

from phoenix.evals.models import OpenAIModel
from phoenix.experiments import evaluate_experiment
from phoenix.experiments.evaluators.llm_evaluators import LLMCriteriaEvaluator

llm_evaluator = LLMCriteriaEvaluator(
    name="is_sql",
    criteria="is_sql",
    description="the output is a valid SQL query and that it executes without errors",
    model=OpenAIModel(),
)

evaluate_experiment(experiment, evaluators=[llm_evaluator])

import json

from pydantic import BaseModel


class Question(BaseModel):
    sql: str
    question: str


class Questions(BaseModel):
    questions: list[Question]


synthetic_data_prompt = f"""\
You are a SQL expert, and you are given a single table named nba with the following columns:

Column | Type | Example
-------|------|--------
{"\n".join(f"{column['column_name']} | {column['column_type']} | {samples[column['column_name']]}" for column in columns)}

Generate SQL queries that would be interesting to ask about this table. Return the SQL query as a string, as well as the
question that the query answers."""

response = await client.chat.completions.create(
    model="gpt-4o",
    temperature=0,
    messages=[
        {
            "role": "user",
            "content": synthetic_data_prompt,
        }
    ],
    tools=[
        {
            "type": "function",
            "function": {
                "name": "generate_questions",
                "description": "Generate SQL queries that would be interesting to ask about this table.",
                "parameters": Questions.model_json_schema(),
            },
        }
    ],
    tool_choice={"type": "function", "function": {"name": "generate_questions"}},
)

generated_questions = json.loads(response.choices[0].message.tool_calls[0].function.arguments)[
    "questions"
]
generated_questions[0]

generated_dataset = []
for q in generated_questions:
    try:
        result = execute_query(q["sql"])
        generated_dataset.append(
            {
                "input": q["question"],
                "expected": {
                    "results": result,
                    "error": None,
                    "query": q["sql"],
                },
                "metadata": {
                    "category": "Generated",
                },
            }
        )
    except duckdb.Error as e:
        print(f"Query failed: {q['sql']}", e)
        print("Skipping...")

generated_dataset[0]

synthetic_dataset = px.Client().upload_dataset(
    dataset_name="nba-golden-synthetic",
    inputs=[{"question": example["input"]} for example in generated_dataset],
    outputs=[example["expected"] for example in generated_dataset],
);

run_experiment(
    synthetic_dataset, task=task, evaluators=[no_error, has_results], experiment_metadata=CONFIG
)

TASK_MODEL = "gpt-3.5-turbo"

experiment = run_experiment(
    synthetic_dataset,
    task=task,
    evaluators=[no_error, has_results],
    experiment_metadata={"model": TASK_MODEL},
)

!pip install -qqqq "arize-phoenix>=8.0.0" datasets openinference-instrumentation-openai

import os
from getpass import getpass

os.environ["PHOENIX_COLLECTOR_ENDPOINT"] = "https://app.phoenix.arize.com"
if not os.environ.get("PHOENIX_CLIENT_HEADERS"):
    os.environ["PHOENIX_CLIENT_HEADERS"] = "api_key=" + getpass("Enter your Phoenix API key: ")

if not os.environ.get("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = getpass("Enter your OpenAI API key: ")

import uuid

from datasets import load_dataset

import phoenix as px
from phoenix.client import Client as PhoenixClient

ds = load_dataset("syeddula/math_word_problems")["train"]
ds = ds.to_pandas()
ds.head()

unique_id = uuid.uuid4()

# Upload the dataset to Phoenix
dataset = px.Client().upload_dataset(
    dataframe=ds,
    input_keys=["Word Problem"],
    output_keys=["Answer"],
    dataset_name=f"wordproblems-{unique_id}",
)

from openai import OpenAI
from openai.types.chat.completion_create_params import CompletionCreateParamsBase

from phoenix.client.types import PromptVersion

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {
            "role": "system",
            "content": "You are an evaluator who outputs the answer to a math word problem. Only respond with the integer answer. Be sure not include words, explanations, symbols, labels, or units and round all decimals answers.",
        },
        {"role": "user", "content": "{{Problem}}"},
    ],
)

prompt_identifier = "wordproblems"

prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="A prompt for computing answers to word problems.",
    version=PromptVersion.from_openai(params),
)

import nest_asyncio

from phoenix.experiments import run_experiment

nest_asyncio.apply()


def zero_shot_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **prompt.format(variables={"Problem": input["Word Problem"]})
    )
    return resp.choices[0].message.content.strip()


def evaluate_response(output, expected):
    if not output.isdigit():
        return False
    return int(output) == int(expected["Answer"])


initial_experiment = run_experiment(
    dataset,
    task=zero_shot_prompt,
    evaluators=[evaluate_response],
    experiment_description="Zero-Shot Prompt",
    experiment_name="zero-shot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + prompt.id},
)

zero_shot_COT_template = """
You are an evaluator who outputs the answer to a math word problem.

You must always think through the problem logically before providing an answer.

First, show some of your reasoning.

Then output the integer answer ONLY on a final new line. In this final answer, be sure not include words, commas, labels, or units and round all decimals answers.

"""

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": zero_shot_COT_template},
        {"role": "user", "content": "{{Problem}}"},
    ],
)

zero_shot_COT = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Zero Shot COT prompt",
    version=PromptVersion.from_openai(params),
)

import re


def zero_shot_COT_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **zero_shot_COT.format(variables={"Problem": input["Word Problem"]})
    )
    response_text = resp.choices[0].message.content.strip()
    lines = response_text.split("\n")
    final_answer = lines[-1].strip()
    final_answer = re.sub(r"^\*\*(\d+)\*\*$", r"\1", final_answer)
    return {"full_response": response_text, "final_answer": final_answer}


def evaluate_response(output, expected):
    final_answer = output["final_answer"]
    if not final_answer.isdigit():
        return False
    return int(final_answer) == int(expected["Answer"])


initial_experiment = run_experiment(
    dataset,
    task=zero_shot_COT_prompt,
    evaluators=[evaluate_response],
    experiment_description="Zero-Shot COT Prompt",
    experiment_name="zero-shot-cot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + zero_shot_COT.id},
)

consistency_COT_template = """

You are an evaluator who outputs the answer to a math word problem.

Follow these steps:
1. Solve the problem **multiple times independently**, thinking through the solution carefully each time.
2. Show some of your reasoning for each independent attempt.
3. Identify the integer answer that appears most frequently across your attempts.
4. On a **new line**, output only this majority answer as a plain integer with **no words, commas, labels, units, or special characters**.
"""

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": consistency_COT_template},
        {"role": "user", "content": "{{Problem}}"},
    ],
)

self_consistency_COT = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="self consistency COT prompt",
    version=PromptVersion.from_openai(params),
)

def self_consistency_COT_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **self_consistency_COT.format(variables={"Problem": input["Word Problem"]})
    )
    response_text = resp.choices[0].message.content.strip()
    lines = response_text.split("\n")
    final_answer = lines[-1].strip()
    final_answer = re.sub(r"^\*\*(\d+)\*\*$", r"\1", final_answer)
    return {"full_response": response_text, "final_answer": final_answer}


def evaluate_response(output, expected):
    final_answer = output["final_answer"]
    if not final_answer.isdigit():
        return False
    return int(final_answer) == int(expected["Answer"])


initial_experiment = run_experiment(
    dataset,
    task=self_consistency_COT_prompt,
    evaluators=[evaluate_response],
    experiment_description="Self Consistency COT Prompt",
    experiment_name="self-consistency-cot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + self_consistency_COT.id},
)

ds = load_dataset("syeddula/math_word_problems")["test"]
few_shot_examples = ds.to_pandas().sample(5)
few_shot_examples

few_shot_COT_template = """
You are an evaluator who outputs the answer to a math word problem. You must always think through the problem logically before providing an answer. Show some of your reasoning.

Finally, output the integer answer ONLY on a final new line. In this final answer, be sure not include words, commas, labels, or units and round all decimals answers.

Here are some examples of word problems, step by step explanations, and solutions to guide your reasoning:

{examples}
"""
params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": few_shot_COT_template.format(examples=few_shot_examples)},
        {"role": "user", "content": "{{Problem}}"},
    ],
)

few_shot_COT = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Few Shot COT prompt",
    version=PromptVersion.from_openai(params),
)

def few_shot_COT_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **few_shot_COT.format(variables={"Problem": input["Word Problem"]})
    )
    response_text = resp.choices[0].message.content.strip()
    lines = response_text.split("\n")
    final_answer = lines[-1].strip()
    final_answer = re.sub(r"^\*\*(\d+)\*\*$", r"\1", final_answer)
    return {"full_response": response_text, "final_answer": final_answer}


def evaluate_response(output, expected):
    final_answer = output["final_answer"]
    if not final_answer.isdigit():
        return False
    return int(final_answer) == int(expected["Answer"])


import nest_asyncio

from phoenix.experiments import run_experiment

nest_asyncio.apply()

initial_experiment = run_experiment(
    dataset,
    task=few_shot_COT_prompt,
    evaluators=[evaluate_response],
    experiment_description="Few-Shot COT Prompt",
    experiment_name="few-shot-cot-prompt",
    experiment_metadata={"prompt": "prompt_id=" + few_shot_COT.id},
)

!pip install -q "arize-phoenix>=8.0.0" datasets openinference-instrumentation-openai

import os
from getpass import getpass

os.environ["PHOENIX_COLLECTOR_ENDPOINT"] = "https://app.phoenix.arize.com"
if not os.environ.get("PHOENIX_CLIENT_HEADERS"):
    os.environ["PHOENIX_CLIENT_HEADERS"] = "api_key=" + getpass("Enter your Phoenix API key: ")

if not os.environ.get("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = getpass("Enter your OpenAI API key: ")

from datasets import load_dataset

ds = load_dataset("syeddula/empathy_scores")["test"]
ds = ds.to_pandas()
ds.head()

import uuid

import phoenix as px

unique_id = uuid.uuid4()

# Upload the dataset to Phoenix
dataset = px.Client().upload_dataset(
    dataframe=ds,
    input_keys=["AI_Response", "EI_Empathy_Score"],
    output_keys=["EI_Empathy_Score"],
    dataset_name=f"empathy-{unique_id}",
)

from openai import OpenAI

client = OpenAI()


def generate_eval_template():
    meta_prompt = """
    You are an expert in AI evaluation and emotional intelligence assessment. Your task is to create a structured evaluation template for assessing the emotional intelligence and empathy of AI responses to user inputs.

    ### Task Overview:
    Generate a detailed evaluation template that measures the AI’s ability to recognize user emotions, respond empathetically, and provide emotionally appropriate responses. The template should:
    - Include 3 to 5 distinct evaluation criteria that assess different aspects of emotional intelligence.
    - Define a scoring system on a scale of 1 to 10, ensuring a broad distribution of scores across different responses.
    - Provide clear, tiered guidelines for assigning scores, distinguishing weak, average, and strong performance.
    - Include a justification section requiring evaluators to explain the assigned score with specific examples.
    - Ensure the scoring rubric considers complexity and edge cases, preventing generic or uniform scores.

    ### Format:
    Return the evaluation template as plain text, structured with headings, criteria, and a detailed scoring rubric. The template should be easy to follow and apply to real-world datasets.

    ### Scoring Guidelines:
    - The scoring system must be on a **scale of 1 to 10** and encourage a full range of scores.
    - Differentiate between strong, average, and weak responses using specific, well-defined levels.
    - Require evaluators to justify scores

    Do not include any concluding remarks such as 'End of Template' or similar statements. The template should end naturally after the final section.

    """

    try:
        response = client.chat.completions.create(
            model="gpt-4",
            messages=[{"role": "user", "content": meta_prompt}],
            temperature=0.9,  # High temperature for more creativity
        )

        return response.choices[0].message.content
    except Exception as e:
        return {"error": str(e)}


print("Generating new evaluation template...")
EMPATHY_EVALUATION_PROMPT_TEMPLATE = generate_eval_template()
print("Template generated successfully!")
print(EMPATHY_EVALUATION_PROMPT_TEMPLATE)

from openinference.instrumentation.openai import OpenAIInstrumentor

from phoenix.otel import register

tracer_provider = register(
    project_name="LLM-as-a-Judge", endpoint="https://app.phoenix.arize.com/v1/traces"
)
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

import pandas as pd

from phoenix.evals import (
    OpenAIModel,
    llm_classify,
)


def initial_prompt(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


def evaluate_response(input, output):
    expected_score = input["EI_Empathy_Score"]
    predicted_score = output
    return abs(expected_score - predicted_score) <= 2

import nest_asyncio

from phoenix.experiments import run_experiment

nest_asyncio.apply()

initial_experiment = run_experiment(
    dataset, task=initial_prompt, evaluators=[evaluate_response], experiment_name="initial_prompt"
)

few_shot_examples = """
---
To help guide your reasoning, compare the text to the following examples and scorings:

Example 1: "You should try to focus and get your work done. Everyone has tough days."
  Score: 2
  Explanation: No acknowledgment of the user’s emotions, dismissive and offers no real support

Example 2: "That sounds really stressful. Have you considered talking to a supervisor or taking breaks?"
  Score: 5
  Explanation: Acknowledges stress, but in a generic way. Provides advice, but not very personal. Could be warmer in tone.

Example 3: "I’m really sorry you’re feeling this way. It’s completely understandable to feel overwhelmed. You’re not alone in this. Have you had a chance to take a break or talk to someone who can support you?"
  Score: 9
  Explanation: Validates emotions, reassures the user, and offers support
"""
EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED = EMPATHY_EVALUATION_PROMPT_TEMPLATE + few_shot_examples

def llm_as_a_judge(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


experiment = run_experiment(
    dataset,
    task=llm_as_a_judge,
    evaluators=[evaluate_response],
    experiment_name="few_shot_examples",
)

style_invariant = """
----
To help guide your reasoning, below is an example of how different response styles and tones can achieve similar scores:

#### Scenario: Customer Support Handling a Late Order
User: "My order is late, and I needed it for an important event. This is really frustrating."

Response A (Formal): "I sincerely apologize for the delay..."
Response B (Casual): "Oh no, that’s really frustrating!..."
Response C (Direct): "Sorry about that. I’ll check..."
"""
EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED = EMPATHY_EVALUATION_PROMPT_TEMPLATE + style_invariant

def llm_as_a_judge(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


experiment = run_experiment(
    dataset, task=llm_as_a_judge, evaluators=[evaluate_response], experiment_name="style_invariant"
)

def generate_condensed_template():
    meta_prompt = """
    You are an expert in prompt engineering and LLM evaluation. Your task is to optimize a given LLM-as-a-judge prompt by reducing its word count significantly while maintaining all essential information, including evaluation criteria, scoring system, and purpose.

    Requirements:
    Preserve all key details such as metrics, scoring guidelines, and judgment criteria.

    Eliminate redundant phrasing and unnecessary explanations.

    Ensure clarity and conciseness without losing meaning.

    Maintain the prompt’s effectiveness for consistent evaluations.

    Output Format:
    Return only the optimized prompt as plain text, with no explanations or commentary.

    """

    try:
        response = client.chat.completions.create(
            model="gpt-4",
            messages=[
                {
                    "role": "user",
                    "content": "Provided LLM-as-a-judge prompt"
                    + EMPATHY_EVALUATION_PROMPT_TEMPLATE,
                },
                {"role": "user", "content": meta_prompt},
            ],
            temperature=0.9,  # High temperature for more creativity
        )

        return response.choices[0].message.content
    except Exception as e:
        return {"error": str(e)}


print("Generating condensed evaluation template...")
EMPATHY_EVALUATION_PROMPT_TEMPLATE_CONDENSED = generate_condensed_template()
print("Template generated successfully!")
print(EMPATHY_EVALUATION_PROMPT_TEMPLATE_CONDENSED)

def llm_as_a_judge(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE_CONDENSED,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


experiment = run_experiment(
    dataset, task=llm_as_a_judge, evaluators=[evaluate_response], experiment_name="condensed_prompt"
)

refinement_text = """
---
After you have done the evaluation, follow these two steps:
1. Self-Critique
Review your initial score:
- Was it too harsh or lenient?
- Did it consider the full context?
- Would others agree with your score?
Explain any inconsistencies briefly.

2. Final Refinement
Based on your critique, adjust your score if necessary.
- Only output a number (1-10)
"""
EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED = EMPATHY_EVALUATION_PROMPT_TEMPLATE + refinement_text

def llm_as_a_judge(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


experiment = run_experiment(
    dataset, task=llm_as_a_judge, evaluators=[evaluate_response], experiment_name="self_refinement"
)

EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED = (
    EMPATHY_EVALUATION_PROMPT_TEMPLATE + few_shot_examples + style_invariant
)

def llm_as_a_judge(input):
    response_classifications = llm_classify(
        dataframe=pd.DataFrame([{"AI_Response": input["AI_Response"]}]),
        template=EMPATHY_EVALUATION_PROMPT_TEMPLATE_IMPROVED,
        model=OpenAIModel(model="gpt-4"),
        rails=list(map(str, range(1, 11))),
        provide_explanation=True,
    )
    score = response_classifications.iloc[0]["label"]
    return int(score)


experiment = run_experiment(
    dataset, task=llm_as_a_judge, evaluators=[evaluate_response], experiment_name="combined"
)

!pip install -q "arize-phoenix>=8.0.0" datasets

import os
from getpass import getpass

os.environ["PHOENIX_COLLECTOR_ENDPOINT"] = "https://app.phoenix.arize.com"
if not os.environ.get("PHOENIX_CLIENT_HEADERS"):
    os.environ["PHOENIX_CLIENT_HEADERS"] = "api_key=" + getpass("Enter your Phoenix API key: ")

if not os.environ.get("OPENAI_API_KEY"):
    os.environ["OPENAI_API_KEY"] = getpass("Enter your OpenAI API key: ")

from datasets import load_dataset

ds = load_dataset("jackhhao/jailbreak-classification")["train"]
ds = ds.to_pandas().sample(50)
ds.head()

import uuid

import phoenix as px
from phoenix.client import Client as PhoenixClient

unique_id = uuid.uuid4()

# Upload the dataset to Phoenix
dataset = px.Client().upload_dataset(
    dataframe=ds,
    input_keys=["prompt"],
    output_keys=["type"],
    dataset_name=f"jailbreak-classification-{unique_id}",
)

from openai import OpenAI
from openai.types.chat.completion_create_params import CompletionCreateParamsBase

from phoenix.client.types import PromptVersion

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {
            "role": "system",
            "content": "You are an evaluator that decides whether a given prompt is a jailbreak risk. Only output 'benign' or 'jailbreak', no other words.",
        },
        {"role": "user", "content": "{{prompt}}"},
    ],
)

prompt_identifier = "jailbreak-classification"

prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="A prompt for classifying whether a given prompt is a jailbreak risk.",
    version=PromptVersion.from_openai(params),
)

def test_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(**prompt.format(variables={"prompt": input["prompt"]}))
    return resp.choices[0].message.content.strip()


def evaluate_response(output, expected):
    return output.lower() == expected["type"].lower()

from openinference.instrumentation.openai import OpenAIInstrumentor

from phoenix.otel import register

tracer_provider = register(project_name="prompt-optimization")
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

import nest_asyncio

from phoenix.experiments import run_experiment

nest_asyncio.apply()

initial_experiment = run_experiment(
    dataset,
    task=test_prompt,
    evaluators=[evaluate_response],
    experiment_description="Initial base prompt",
    experiment_name="initial-prompt",
    experiment_metadata={"prompt": "prompt_id=" + prompt.id},
)

from datasets import load_dataset

ds_test = load_dataset("jackhhao/jailbreak-classification")[
    "test"
]  # this time, load in the test set instead of the training set
few_shot_examples = ds_test.to_pandas().sample(10)

few_shot_template = """
You are an evaluator that decides whether a given prompt is a jailbreak risk. Only output "benign" or "jailbreak", no other words.

Here are some examples of prompts and responses:

{examples}
"""

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": few_shot_template.format(examples=few_shot_examples)},
        {"role": "user", "content": "{{prompt}}"},
    ],
)

few_shot_prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Few shot prompt",
    version=PromptVersion.from_openai(params),
)

def test_prompt(input):
    client = OpenAI()
    prompt_vars = {"prompt": input["prompt"]}
    resp = client.chat.completions.create(**few_shot_prompt.format(variables=prompt_vars))
    return resp.choices[0].message.content.strip()

few_shot_experiment = run_experiment(
    dataset,
    task=test_prompt,
    evaluators=[evaluate_response],
    experiment_description="Prompt Optimization Technique #1: Few Shot Examples",
    experiment_name="few-shot-examples",
    experiment_metadata={"prompt": "prompt_id=" + few_shot_prompt.id},
)

# Access the experiment results from the first round as a dataframe
ground_truth_df = initial_experiment.as_dataframe()

# Sample 10 examples to use as meta prompting examples
ground_truth_df = ground_truth_df[:10]

# Create a new column with the examples in a single string
ground_truth_df["example"] = ground_truth_df.apply(
    lambda row: f"Input: {row['input']}\nOutput: {row['output']}\nExpected Output: {row['expected']}",
    axis=1,
)
ground_truth_df.head()

meta_prompt = """
You are an expert prompt engineer. You are given a prompt, and a list of examples.

Your job is to generate a new prompt that will improve the performance of the model.

Here are the examples:

{examples}

Here is the original prompt:

{prompt}

Here is the new prompt:
"""

original_base_prompt = (
    prompt.format(variables={"prompt": "example prompt"}).get("messages")[0].get("content")
)

client = OpenAI()
response = client.chat.completions.create(
    model="gpt-3.5-turbo",
    messages=[
        {
            "role": "user",
            "content": meta_prompt.format(
                prompt=original_base_prompt, examples=ground_truth_df["example"].to_string()
            ),
        }
    ],
)
new_prompt = response.choices[0].message.content.strip()

new_prompt

if r"\{examples\}" in new_prompt:
    new_prompt = new_prompt.format(examples=few_shot_examples)

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {"role": "system", "content": new_prompt},
        {"role": "user", "content": "{{prompt}}"},
    ],
)

meta_prompt_result = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Meta prompt result",
    version=PromptVersion.from_openai(params),
)

def test_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **meta_prompt_result.format(variables={"prompt": input["prompt"]})
    )
    return resp.choices[0].message.content.strip()

meta_prompting_experiment = run_experiment(
    dataset,
    task=test_prompt,
    evaluators=[evaluate_response],
    experiment_description="Prompt Optimization Technique #2: Meta Prompting",
    experiment_name="meta-prompting",
    experiment_metadata={"prompt": "prompt_id=" + meta_prompt_result.id},
)

import numpy as np


# First we'll define a function to get embeddings for prompts
def get_embedding(text):
    client = OpenAI()
    response = client.embeddings.create(model="text-embedding-ada-002", input=text)
    return response.data[0].embedding


# Function to calculate gradient direction between successful and failed prompts
def calculate_prompt_gradient(successful_prompts, failed_prompts):
    # Get embeddings for successful and failed prompts
    successful_embeddings = [get_embedding(p) for p in successful_prompts]
    failed_embeddings = [get_embedding(p) for p in failed_prompts]

    # Calculate average embeddings
    avg_successful = np.mean(successful_embeddings, axis=0)
    avg_failed = np.mean(failed_embeddings, axis=0)

    # Calculate gradient direction
    gradient = avg_successful - avg_failed
    return gradient / np.linalg.norm(gradient)


# Get successful and failed examples from our dataset
successful_examples = (
    ground_truth_df[ground_truth_df["output"] == ground_truth_df["expected"].get("type")]["input"]
    .apply(lambda x: x["prompt"])
    .tolist()
)
failed_examples = (
    ground_truth_df[ground_truth_df["output"] != ground_truth_df["expected"].get("type")]["input"]
    .apply(lambda x: x["prompt"])
    .tolist()
)

# Calculate the gradient direction
gradient = calculate_prompt_gradient(successful_examples[:5], failed_examples[:5])


# Function to optimize a prompt using the gradient
def optimize_prompt(base_prompt, gradient, step_size=0.1):
    # Get base embedding
    base_embedding = get_embedding(base_prompt)

    # Move in gradient direction
    optimized_embedding = base_embedding + step_size * gradient

    # Use GPT to convert the optimized embedding back to text
    client = OpenAI()
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[
            {
                "role": "system",
                "content": "You are helping to optimize prompts. Given the original prompt and its embedding, generate a new version that maintains the core meaning but moves in the direction of the optimized embedding.",
            },
            {
                "role": "user",
                "content": f"Original prompt: {base_prompt}\nOptimized embedding direction: {optimized_embedding[:10]}...\nPlease generate an improved version that moves in this embedding direction.",
            },
        ],
    )
    return response.choices[0].message.content.strip()


# Test the gradient-based optimization
gradient_prompt = optimize_prompt(original_base_prompt, gradient)

gradient_prompt

if r"\{examples\}" in gradient_prompt:
    gradient_prompt = gradient_prompt.format(examples=few_shot_examples)

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {
            "role": "system",
            "content": gradient_prompt,
        },  # if your meta prompt includes few shot examples, make sure to include them here
        {"role": "user", "content": "{{prompt}}"},
    ],
)

gradient_prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="Gradient prompt result",
    version=PromptVersion.from_openai(params),
)

def test_gradient_prompt(input):
    client = OpenAI()
    resp = client.chat.completions.create(
        **gradient_prompt.format(variables={"prompt": input["prompt"]})
    )
    return resp.choices[0].message.content.strip()

gradient_experiment = run_experiment(
    dataset,
    task=test_gradient_prompt,
    evaluators=[evaluate_response],
    experiment_description="Prompt Optimization Technique #3: Prompt Gradients",
    experiment_name="gradient-optimization",
    experiment_metadata={"prompt": "prompt_id=" + gradient_prompt.id},
)

!pip install -q dspy openinference-instrumentation-dspy

from openinference.instrumentation.dspy import DSPyInstrumentor

DSPyInstrumentor().instrument(tracer_provider=tracer_provider)

# Import DSPy and set up the language model
import dspy

# Configure DSPy to use OpenAI
turbo = dspy.LM(model="gpt-3.5-turbo")
dspy.settings.configure(lm=turbo)


# Define the prompt classification task
class PromptClassifier(dspy.Signature):
    """Classify if a prompt is benign or jailbreak."""

    prompt = dspy.InputField()
    label = dspy.OutputField(desc="either 'benign' or 'jailbreak'")


# Create the basic classifier
classifier = dspy.Predict(PromptClassifier)

classifier(prompt=ds.iloc[0].prompt)

def validate_classification(example, prediction, trace=None):
    return example["label"] == prediction["label"]


# Prepare training data from previous examples
train_data = []
for _, row in ground_truth_df.iterrows():
    example = dspy.Example(
        prompt=row["input"]["prompt"], label=row["expected"]["type"]
    ).with_inputs("prompt")
    train_data.append(example)

tp = dspy.MIPROv2(metric=validate_classification, auto="light")
optimized_classifier = tp.compile(classifier, trainset=train_data)

params = CompletionCreateParamsBase(
    model="gpt-3.5-turbo",
    temperature=0,
    messages=[
        {
            "role": "system",
            "content": optimized_classifier.signature.instructions,
        },  # if your meta prompt includes few shot examples, make sure to include them here
        {"role": "user", "content": "{{prompt}}"},
    ],
)

dspy_prompt = PhoenixClient().prompts.create(
    name=prompt_identifier,
    prompt_description="DSPy prompt result",
    version=PromptVersion.from_openai(params),
)

# Create evaluation function using optimized classifier
def test_dspy_prompt(input):
    result = optimized_classifier(prompt=input["prompt"])
    return result.label

# Run experiment with DSPy-optimized classifier
dspy_experiment = run_experiment(
    dataset,
    task=test_dspy_prompt,
    evaluators=[evaluate_response],
    experiment_description="Prompt Optimization Technique #4: DSPy Prompt Tuning",
    experiment_name="dspy-optimization",
    experiment_metadata={"prompt": "prompt_id=" + dspy_prompt.id},
)

prompt_gen_lm = dspy.LM("gpt-4o")
tp = dspy.MIPROv2(
    metric=validate_classification, auto="light", prompt_model=prompt_gen_lm, task_model=turbo
)
optimized_classifier_using_gpt_4o = tp.compile(classifier, trainset=train_data)

# Create evaluation function using optimized classifier
def test_dspy_prompt(input):
    result = optimized_classifier_using_gpt_4o(prompt=input["prompt"])
    return result.label

# Run experiment with DSPy-optimized classifier
dspy_experiment_using_gpt_4o = run_experiment(
    dataset,
    task=test_dspy_prompt,
    evaluators=[evaluate_response],
    experiment_description="Prompt Optimization Technique #5: DSPy Prompt Tuning with GPT-4o",
    experiment_name="dspy-optimization-gpt-4o",
    experiment_metadata={"prompt": "prompt_id=" + dspy_prompt.id},
)