Identifying High-Signal Traces in Production

# Find traces with errors in the last hour (--limit defaults to 10, newest-first,
# so raise it or you'll only scan the 10 most recent traces once volume climbs)
px trace list --last-n-minutes 60 --limit 200 --format raw --no-progress | jq '.[] | select(.status == "ERROR")'

# List all errored spans
px span list --status-code ERROR --limit 20

# Only LLM-call failures (use --span-kind TOOL for tool errors)
px span list --status-code ERROR --span-kind LLM --limit 20

# Drill down into a specific trace
px trace get <trace-id> --format raw | jq '.spans[] | select(.status_code != "OK")'

# Fetch a wide window (--limit defaults to 10), then sort and slice the extremes.
# Find the slowest traces (descending by duration)
px trace list --limit 200 --format raw --no-progress | jq 'sort_by(-.duration) | .[0:10]'

# Find suspiciously fast traces (ascending by duration)
px trace list --limit 200 --format raw --no-progress | jq 'sort_by(.duration) | .[0:10]'

from phoenix.client import Client
from phoenix.trace.dsl import SpanQuery
import pandas as pd

client = Client()
PROJECT = "high-signal-demo"

# Pull all LLM spans with timing and token data (the dataframe is indexed by span_id)
perf_query = SpanQuery().where("span_kind == 'LLM'").select(
    trace_id="context.trace_id",
    input="input.value",
    model="llm.model_name",
    start_time="start_time",
    end_time="end_time",
    prompt_tokens="llm.token_count.prompt",
    completion_tokens="llm.token_count.completion",
    total_tokens="llm.token_count.total",
)
df = client.spans.get_spans_dataframe(query=perf_query, project_name=PROJECT)

# Calculate duration in milliseconds
df['duration'] = (pd.to_datetime(df['end_time']) - pd.to_datetime(df['start_time'])).dt.total_seconds() * 1000
df['input'] = df['input'].str.slice(0, 60)  # trim for display

# Compute percentiles
p95_duration = df['duration'].quantile(0.95)
p5_duration = df['duration'].quantile(0.05)
p95_tokens = df['total_tokens'].quantile(0.95)

cols = ['context.trace_id', 'duration', 'total_tokens', 'input']

# Slow tail (potential loops, retries): grab the trace_id to go read it
slow_tail = df[df['duration'] > p95_duration].sort_values('duration', ascending=False)
print(f"Slow tail (>{p95_duration:.0f}ms): {len(slow_tail)} spans")
print(slow_tail[cols].to_string(), "\n")

# Fast tail (potential silent failures)
fast_tail = df[df['duration'] < p5_duration].sort_values('duration')
print(f"Fast tail (<{p5_duration:.0f}ms): {len(fast_tail)} spans")
print(fast_tail[cols].to_string(), "\n")

# High token usage (cost outliers)
high_cost = df[df['total_tokens'] > p95_tokens].sort_values('total_tokens', ascending=False)
print(f"High token usage (>{p95_tokens:.0f} tokens): {len(high_cost)} spans")
print(high_cost[cols].to_string())

# Drill into any of these from the terminal:  px trace get <trace_id>

import { createClient } from "@arizeai/phoenix-client";
import { getSpans } from "@arizeai/phoenix-client/spans";

const client = createClient();

async function main() {
  const { spans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    spanKind: "LLM",
    limit: 1000, // Adjust based on volume
  });

  const rows = spans
    .map((span) => ({
      traceId: span.context.trace_id,
      spanId: span.context.span_id,
      durationMs:
        new Date(span.end_time).getTime() - new Date(span.start_time).getTime(),
      tokens: span.attributes["llm.token_count.total"] as number,
      input: String(span.attributes["input.value"] ?? "").slice(0, 60),
    }))
    .sort((a, b) => b.durationMs - a.durationMs);

  // Slow tail (top 5%): print trace ids so you can go read them
  const p95Index = Math.max(1, Math.floor(rows.length * 0.05));
  const slowTail = rows.slice(0, p95Index);
  console.log(`Slow tail: ${slowTail.length} spans`);
  console.table(slowTail);

  // Fast tail (bottom 5%)
  const fastTail = rows.slice(-p95Index);
  console.log(`Fast tail: ${fastTail.length} spans`);
  console.table(fastTail);

  // Drill into any of these from the terminal:  px trace get <traceId>
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

from phoenix.client import Client
from phoenix.trace.dsl import SpanQuery

client = Client()
PROJECT = "high-signal-demo"

# Find incorrect answers (filter on the label, not the score).
# Note: filter on labels in .where(); .select() projects span attributes.
# Eval labels can't be projected in .select(), but the .where() filter already
# guarantees the label, so just pull the trace_id + input to go read them.
incorrect = SpanQuery().where(
    "evals['correctness'].label == 'incorrect'"
).select(
    trace_id="context.trace_id",
    input="input.value",
)
incorrect_df = client.spans.get_spans_dataframe(query=incorrect, project_name=PROJECT)
print(f"Found {len(incorrect_df)} spans labeled incorrect (index is span_id):")
print(incorrect_df.assign(input=incorrect_df["input"].str.slice(0, 60)).head(10).to_string(), "\n")

# ALSO spot-check the passing labels to validate your evaluator
correct = SpanQuery().where(
    "evals['correctness'].label == 'correct'"
).select(
    trace_id="context.trace_id",
    input="input.value",
)
correct_df = client.spans.get_spans_dataframe(query=correct, project_name=PROJECT)
# Read a handful: are these actually correct? If not, your judge is miscalibrated.
print(f"{len(correct_df)} spans labeled correct and spot-check a few for false negatives\n")

# Compound signal: incorrect answers that ALSO drew negative user feedback
compound = SpanQuery().where(
    "evals['correctness'].label == 'incorrect' "
    "and evals['user_feedback'].label == '👎'"
).select(
    trace_id="context.trace_id",
    input="input.value",
)
compound_df = client.spans.get_spans_dataframe(query=compound, project_name=PROJECT)
print(f"Incorrect AND thumbs-down: {len(compound_df)} spans")
print(compound_df.assign(input=compound_df["input"].str.slice(0, 60)).to_string())

# Drill into any of these from the terminal:  px trace get <trace_id>

import { createClient } from "@arizeai/phoenix-client";
import { getSpans, getSpanAnnotations } from "@arizeai/phoenix-client/spans";

const client = createClient();

async function main() {
  // Get spans with annotations
  const { spans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    limit: 1000,
  });

  const spanIds = spans.map((s) => s.context.span_id);
  const traceOf = new Map(spans.map((s) => [s.context.span_id, s.context.trace_id]));

  // getSpanAnnotations returns { annotations } and is cursor-paginated (default
  // page size 100), so loop until there's no cursor or you'll miss rows.
  // Each annotation's label is nested under `.result.label` (not `.label`).
  const annotations = [];
  let cursor: string | undefined;
  do {
    const page = await getSpanAnnotations({
      client,
      project: { projectName: "high-signal-demo" },
      spanIds,
      includeAnnotationNames: ["correctness", "user_feedback"],
      cursor,
      limit: 1000,
    });
    annotations.push(...page.annotations);
    cursor = page.nextCursor || undefined;
  } while (cursor);

  // Filter on labels, not score thresholds (score direction varies by evaluator)
  const incorrect = annotations.filter(
    (ann) => ann.name === "correctness" && ann.result?.label === "incorrect"
  );
  console.log(`Spans labeled incorrect: ${incorrect.length}`);
  console.table(
    incorrect.slice(0, 10).map((a) => ({ spanId: a.span_id, traceId: traceOf.get(a.span_id) }))
  );

  // Spot-check the passing labels too (calibration check)
  const correct = annotations.filter(
    (ann) => ann.name === "correctness" && ann.result?.label === "correct"
  );
  console.log(
    `Spans labeled correct: ${correct.length} (read a few to catch false negatives)`
  );

  // Human signal: turns that drew a thumbs-down
  const thumbsDown = annotations.filter(
    (ann) => ann.name === "user_feedback" && ann.result?.label === "👎"
  );
  console.log(`Spans with negative user feedback: ${thumbsDown.length}`);

  // Drill into any of these from the terminal:  px trace get <traceId>
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

from phoenix.client import Client
from phoenix.trace.dsl import SpanQuery

client = Client()
PROJECT = "high-signal-demo"

# Find all spans in sessions with errors
error_sessions = SpanQuery().where(
    "status_code == 'ERROR' and session.id is not None"
).select(
    session_id="session.id",
    trace_id="context.trace_id",
    error="exception.message",
)
error_sessions_df = client.spans.get_spans_dataframe(query=error_sessions, project_name=PROJECT)

# Get the session IDs with errors (feed one to `px session get <id>`)
problem_sessions = error_sessions_df['session_id'].unique()
print(f"Found {len(problem_sessions)} sessions with errors: {list(problem_sessions)}\n")

# Find long sessions (high turn count = user struggling)
all_sessions = SpanQuery().where(
    "session.id is not None"
).select(
    session_id="session.id",
    trace_id="context.trace_id",
)
sessions_df = client.spans.get_spans_dataframe(query=all_sessions, project_name=PROJECT)

# Count traces per session (context.* projections keep their dotted column name)
# and flag which sessions contained an error, so you can see both signals at once.
turn_counts = sessions_df.groupby('session_id')['context.trace_id'].nunique()
summary = turn_counts.rename('turn_count').reset_index()
summary['has_errors'] = summary['session_id'].isin(problem_sessions)
summary = summary.sort_values('turn_count', ascending=False)

print("Longest sessions:")
print(summary.head(5).to_string(index=False))

high_turn_sessions = summary[summary['turn_count'] > turn_counts.quantile(0.90)]
print(f"\nLong sessions (>90th percentile): {len(high_turn_sessions)}")

import { createClient } from "@arizeai/phoenix-client";
import { getSpans } from "@arizeai/phoenix-client/spans";
import { listSessions } from "@arizeai/phoenix-client/sessions";

const client = createClient();

async function main() {
  // Sessions containing any error trace: pull the error spans (server-side
  // status filter) and collect the session.id they belong to.
  const { spans: errorSpans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    statusCode: "ERROR",
    limit: 1000,
  });
  const errorSessionIds = new Set(
    errorSpans.map((s) => s.attributes["session.id"]).filter(Boolean)
  );
  console.log(`Sessions with errors: ${errorSessionIds.size}`, [...errorSessionIds]);

  // Longest sessions (high turn count = user struggling). listSessions gives the
  // turn count directly as traces.length. No need to reconstruct it from spans.
  const sessions = await listSessions({ client, project: "high-signal-demo" });
  const longest = sessions
    .map((s) => ({
      sessionId: s.sessionId,
      turnCount: s.traces.length,
      hasErrors: errorSessionIds.has(s.sessionId),
    }))
    .sort((a, b) => b.turnCount - a.turnCount);
  console.log("Longest sessions:");
  console.table(longest.slice(0, 5));

  // Drill into one with:  px session get <sessionId>
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

// TypeScript: Export spans, then cluster in Python or use an external tool
// The pattern: getSpans → write to JSON → cluster offline → rejoin results

import { createClient } from "@arizeai/phoenix-client";
import { getSpans } from "@arizeai/phoenix-client/spans";
import * as fs from "fs";

const client = createClient();

async function main() {
  // Export spans for offline clustering
  const { spans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    limit: 1000, // REST caps limit at 1000; paginate with nextCursor for more
  });

  // Extract inputs for embedding
  const inputs = spans.map((span) => ({
    span_id: span.context.span_id,
    input: span.attributes["input.value"],
    status: span.status_code,
  }));

  fs.writeFileSync("spans_for_clustering.json", JSON.stringify(inputs, null, 2));

  console.log(
    "Exported spans to spans_for_clustering.json. Next: cluster with Python, then rejoin"
  );
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});

import { createClient } from "@arizeai/phoenix-client";
import { getSpans } from "@arizeai/phoenix-client/spans";
import * as fs from "fs";

const client = createClient();

async function main() {
  // Time range: last 7 days
  const endTime = new Date();
  const startTime = new Date(endTime.getTime() - 7 * 24 * 60 * 60 * 1000);

  console.log("Weekly Trace Review: Mixed Sample");
  console.log(`Period: ${startTime.toDateString()} to ${endTime.toDateString()}\n`);

  // 1. All hard failures (status filtered server-side)
  const { spans: errorSpans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    statusCode: "ERROR",
    startTime,
    endTime,
    limit: 1000,
  });
  console.log(`Hard failures: ${errorSpans.length}`);

  // 2. Get all spans for tail analysis
  const { spans: allSpans } = await getSpans({
    client,
    project: { projectName: "high-signal-demo" },
    startTime,
    endTime,
    limit: 1000, // REST caps limit at 1000; paginate with nextCursor for more
  });

  // Calculate durations
  const withDurations = allSpans.map((span) => ({
    ...span,
    duration:
      new Date(span.end_time).getTime() - new Date(span.start_time).getTime(),
  }));

  // Sort by duration
  const sorted = withDurations.sort((a, b) => b.duration - a.duration);
  const p95Index = Math.floor(sorted.length * 0.05);
  const p5Index = Math.floor(sorted.length * 0.95);

  const slowTail = sorted.slice(0, p95Index);
  const fastTail = sorted.slice(p5Index);

  console.log(`Slow tail (p95+): ${slowTail.length}`);
  console.log(`Fast tail (p5-): ${fastTail.length}`);

  // 3. Random sample (OK spans only)
  const okSpans = allSpans.filter((s) => s.status_code === "OK");
  const randomSample = okSpans
    .sort(() => Math.random() - 0.5)
    .slice(0, Math.min(30, okSpans.length));

  console.log(`Random sample: ${randomSample.length}`);

  // Combine into review queue
  const reviewQueue = [
    ...errorSpans.slice(0, 20),
    ...slowTail.slice(0, 10),
    ...fastTail.slice(0, 5),
    ...randomSample,
  ];

  console.log(`\nTotal review queue: ${reviewQueue.length} spans`);

  // Export for offline review
  fs.writeFileSync(
    "weekly_review_sample.json",
    JSON.stringify(
      reviewQueue.map((s) => ({
        span_id: s.context.span_id,
        name: s.name,
        status: s.status_code,
        input: s.attributes["input.value"],
      })),
      null,
      2
    )
  );

  console.log("Exported to weekly_review_sample.json");
  console.log("Next: Read traces and add notes with:");
  console.log("  px span add-note <span-id> --text 'your observation'");
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});