6. A/B Testing Prompts

Description

Comparing prompt versions with robust statistical metrics. Sample size calculation, statistical significance, and effect size. Metrics to track: accuracy, latency, cost, user satisfaction. Practical implementation and production patterns.

What Is A/B Testing for Prompts?

A/B testing is the method for determining, with statistical certainty, which of two prompt versions produces better results.

Without A/B testing:
"Prompt B looks better" → Deploy → 50% chance of making production worse

With A/B testing:
Prompt B has 94.2% accuracy vs 91.8% for A (p=0.02, significant improvement) → Deploy with confidence

When to Run an A/B Test

Essential Statistical Concepts

The A/B Test Hypothesis

H₀ (Null): There is no difference between prompt A and prompt B
H₁ (Alternative): Prompt B is better than prompt A

To reject H₀ we need:
- p-value < 0.05 (95% confidence)
- A sufficient effect size (a real improvement, not a trivial one)
- An adequate sample size to detect the expected improvement

Types of Error

Sample Size Calculation

Sample size is the most ignored part of A/B testing — and it's what invalidates most tests.

from scipy import stats
import math

def calculate_sample_size(
    baseline_rate: float,
    min_expected_improvement: float,
    alpha: float = 0.05,
    power: float = 0.80
) -> dict:
    """
    Calculates the sample size needed per variant.
    
    baseline_rate: Current proportion (e.g. 0.91 = 91% accuracy)
    min_expected_improvement: The improvement you want to detect (e.g. 0.03 = 3%)
    alpha: Significance level (typical: 0.05)
    power: Test power (typical: 0.80)
    
    Returns: n per variant
    """
    p1 = baseline_rate
    p2 = baseline_rate + min_expected_improvement
    
    # Z-scores
    z_alpha = stats.norm.ppf(1 - alpha / 2)  # Two-tailed
    z_beta = stats.norm.ppf(power)
    
    # Pooled proportion
    p_pool = (p1 + p2) / 2
    
    # Sample size formula for proportions
    numerator = (z_alpha * math.sqrt(2 * p_pool * (1 - p_pool)) + 
                 z_beta * math.sqrt(p1 * (1 - p1) + p2 * (1 - p2))) ** 2
    denominator = (p2 - p1) ** 2
    
    n = math.ceil(numerator / denominator)
    
    return {
        "n_per_variant": n,
        "total_n": n * 2,
        "baseline": f"{p1:.1%}",
        "target": f"{p2:.1%}",
        "improvement_to_detect": f"{min_expected_improvement:.1%}",
        "alpha": alpha,
        "power": power
    }


# Examples:
print("Detect a 3% improvement:")
r = calculate_sample_size(baseline_rate=0.91, min_expected_improvement=0.03)
print(f"  n per variant: {r['n_per_variant']}")
print(f"  total n: {r['total_n']}")

print("\nDetect a 1% improvement:")
r = calculate_sample_size(baseline_rate=0.91, min_expected_improvement=0.01)
print(f"  n per variant: {r['n_per_variant']}")

# Typical output:
# Detect a 3% improvement: n=~350 per variant (700 total)
# Detect a 1% improvement: n=~3000 per variant (6000 total)

Implementing the A/B Test

import random
import time
from openai import OpenAI
from scipy import stats
from dataclasses import dataclass, field
from typing import Callable

client = OpenAI()

@dataclass
class ABTestResult:
    """Full result of an A/B test."""
    prompt_a_name: str
    prompt_b_name: str
    n_a: int
    n_b: int
    
    # Accuracy metrics
    scores_a: list[float] = field(default_factory=list)
    scores_b: list[float] = field(default_factory=list)
    
    # Performance metrics
    latencies_a: list[float] = field(default_factory=list)
    latencies_b: list[float] = field(default_factory=list)
    
    # Cost metrics
    tokens_a: list[int] = field(default_factory=list)
    tokens_b: list[int] = field(default_factory=list)
    
    @property
    def mean_a(self) -> float:
        return sum(self.scores_a) / len(self.scores_a) if self.scores_a else 0.0
    
    @property
    def mean_b(self) -> float:
        return sum(self.scores_b) / len(self.scores_b) if self.scores_b else 0.0
    
    @property
    def relative_improvement(self) -> float:
        if self.mean_a == 0:
            return 0.0
        return (self.mean_b - self.mean_a) / self.mean_a
    
    def statistical_test(self) -> dict:
        """Applies a t-test to determine statistical significance."""
        if len(self.scores_a) < 2 or len(self.scores_b) < 2:
            return {"error": "Not enough data for a statistical test"}
        
        t_stat, p_value = stats.ttest_ind(self.scores_a, self.scores_b)
        
        return {
            "t_statistic": t_stat,
            "p_value": p_value,
            "significant": p_value < 0.05,
            "confidence": f"{(1 - p_value) * 100:.1f}%"
        }
    
    def winner(self) -> str:
        """Determines the winner based on statistical significance."""
        test = self.statistical_test()
        
        if "error" in test:
            return "INCONCLUSIVE"
        
        if not test["significant"]:
            return "TIE (not significant)"
        
        return "B" if self.mean_b > self.mean_a else "A"
    
    def summary(self) -> dict:
        """Full summary of the A/B test."""
        test_stats = self.statistical_test()
        
        return {
            "winner": self.winner(),
            "mean_a": self.mean_a,
            "mean_b": self.mean_b,
            "absolute_improvement": self.mean_b - self.mean_a,
            "relative_improvement": self.relative_improvement,
            "n_a": self.n_a,
            "n_b": self.n_b,
            "p_value": test_stats.get("p_value"),
            "significant": test_stats.get("significant"),
            "latency_p50_a": sorted(self.latencies_a)[len(self.latencies_a)//2] if self.latencies_a else None,
            "latency_p50_b": sorted(self.latencies_b)[len(self.latencies_b)//2] if self.latencies_b else None,
            "avg_tokens_a": sum(self.tokens_a)/len(self.tokens_a) if self.tokens_a else None,
            "avg_tokens_b": sum(self.tokens_b)/len(self.tokens_b) if self.tokens_b else None,
        }


def ab_test_offline(
    prompt_a: str,
    prompt_b: str,
    golden_set: list[dict],
    evaluator: Callable | None = None,
    n_per_variant: int | None = None
) -> ABTestResult:
    """
    Runs an offline A/B test using the golden set.
    
    evaluator: function(expected, actual) -> float (0.0-1.0)
               If None, uses exact match.
    n_per_variant: Maximum number of examples per variant.
                   If None, uses the whole golden set for both.
    """
    if evaluator is None:
        def evaluator(expected, actual):
            return 1.0 if str(expected).strip().lower() == str(actual).strip().lower() else 0.0
    
    # Select the sample
    if n_per_variant and len(golden_set) > n_per_variant:
        sample = random.sample(golden_set, min(len(golden_set), n_per_variant * 2))
        sample_a = sample[:n_per_variant]
        sample_b = sample[n_per_variant:]
    else:
        # If the golden set is small, use every example for both
        sample_a = golden_set
        sample_b = golden_set
    
    result = ABTestResult(
        prompt_a_name="Prompt A",
        prompt_b_name="Prompt B",
        n_a=len(sample_a),
        n_b=len(sample_b)
    )
    
    # Evaluate Prompt A
    print(f"Evaluating Prompt A ({len(sample_a)} examples)...")
    for example in sample_a:
        start = time.time()
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": prompt_a.format(input=example["input"])}],
            temperature=0
        )
        latency = (time.time() - start) * 1000
        output = response.choices[0].message.content.strip()
        
        score = evaluator(example["expected_output"], output)
        result.scores_a.append(score)
        result.latencies_a.append(latency)
        result.tokens_a.append(response.usage.total_tokens)
    
    # Evaluate Prompt B
    print(f"Evaluating Prompt B ({len(sample_b)} examples)...")
    for example in sample_b:
        start = time.time()
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": prompt_b.format(input=example["input"])}],
            temperature=0
        )
        latency = (time.time() - start) * 1000
        output = response.choices[0].message.content.strip()
        
        score = evaluator(example["expected_output"], output)
        result.scores_b.append(score)
        result.latencies_b.append(latency)
        result.tokens_b.append(response.usage.total_tokens)
    
    return result

Multiple Metrics in A/B Testing

In production, you rarely decide based on a single metric. You need a multi-metric framework:

def ab_test_multimetric(
    prompt_a: str,
    prompt_b: str,
    golden_set: list[dict],
    metrics: list[str] = None
) -> dict:
    """
    A/B test evaluating multiple dimensions at the same time.
    
    metrics: List of metrics to evaluate.
             Options: "accuracy", "faithfulness", "format", "latency", "cost"
    """
    if metrics is None:
        metrics = ["accuracy", "format", "latency", "cost"]
    
    results_a = {m: [] for m in metrics}
    results_b = {m: [] for m in metrics}
    
    for example in golden_set:
        for prompt, results in [(prompt_a, results_a), (prompt_b, results_b)]:
            start = time.time()
            
            response = client.chat.completions.create(
                model="gpt-4o-mini",
                messages=[{"role": "user", "content": prompt.format(input=example["input"])}],
                temperature=0
            )
            
            latency_ms = (time.time() - start) * 1000
            output = response.choices[0].message.content.strip()
            
            # Compute each metric
            if "accuracy" in metrics:
                is_correct = output.lower() == str(example["expected_output"]).lower()
                results["accuracy"].append(1.0 if is_correct else 0.0)
            
            if "format" in metrics:
                import json
                try:
                    json.loads(output)
                    format_ok = 1.0
                except:
                    format_ok = 0.0 if example.get("requires_json") else 1.0
                results["format"].append(format_ok)
            
            if "latency" in metrics:
                results["latency"].append(latency_ms)
            
            if "cost" in metrics:
                # GPT-4o-mini: $0.15/1M input + $0.60/1M output
                cost = (response.usage.prompt_tokens * 0.15 / 1e6 + 
                        response.usage.completion_tokens * 0.60 / 1e6)
                results["cost"].append(cost)
    
    # Consolidate the results per metric
    comparison = {}
    for metric in metrics:
        vals_a = results_a[metric]
        vals_b = results_b[metric]
        
        mean_a = sum(vals_a) / len(vals_a)
        mean_b = sum(vals_b) / len(vals_b)
        
        # Statistical test
        if len(vals_a) >= 2:
            _, p_value = stats.ttest_ind(vals_a, vals_b)
        else:
            p_value = 1.0
        
        # For latency and cost, "lower is better"; for the rest, "higher is better"
        if metric in ["latency", "cost"]:
            winner = "B" if mean_b < mean_a else "A"
        else:
            winner = "B" if mean_b > mean_a else "A"
        
        comparison[metric] = {
            "mean_a": mean_a,
            "mean_b": mean_b,
            "delta": mean_b - mean_a,
            "p_value": p_value,
            "significant": p_value < 0.05,
            "winner": winner if p_value < 0.05 else "TIE"
        }
    
    # Determine the overall winner
    wins_a = sum(1 for m in comparison.values() if m["winner"] == "A")
    wins_b = sum(1 for m in comparison.values() if m["winner"] == "B")
    
    if wins_b > wins_a:
        overall_winner = "B"
    elif wins_a > wins_b:
        overall_winner = "A"
    else:
        overall_winner = "TIE"
    
    return {
        "winner": overall_winner,
        "metrics": comparison,
        "wins": {"A": wins_a, "B": wins_b}
    }

A/B Testing in Production (Online)

Offline A/B testing is the most practical option for prompts, but in production you can also run online A/B testing:

import random
from datetime import datetime

class ABRouter:
    """
    Router that splits traffic between two versions of a prompt.
    Use it for A/B testing in production (online).
    """
    
    def __init__(
        self,
        prompt_a: str,
        prompt_b: str,
        split: float = 0.5,
        experiment_id: str = "exp_001"
    ):
        self.prompt_a = prompt_a
        self.prompt_b = prompt_b
        self.split = split
        self.experiment_id = experiment_id
        
        # Result storage
        self.results: list[dict] = []
    
    def get_variant(self, user_id: str | None = None) -> tuple[str, str]:
        """
        Determines which variant to use.
        If user_id is provided, it hashes for consistency (same user = same variant).
        
        Returns: (prompt, variant_name)
        """
        if user_id:
            # Deterministic hashing: the same user always sees the same variant
            import hashlib
            hash_val = int(hashlib.md5(f"{self.experiment_id}:{user_id}".encode()).hexdigest(), 16)
            use_b = (hash_val % 100) < (self.split * 100)
        else:
            use_b = random.random() < self.split
        
        if use_b:
            return self.prompt_b, "B"
        else:
            return self.prompt_a, "A"
    
    def log_result(
        self,
        variant: str,
        input_text: str,
        output: str,
        expected: str | None = None,
        user_feedback: float | None = None
    ) -> None:
        """Records the result of a call for later analysis."""
        self.results.append({
            "timestamp": datetime.now().isoformat(),
            "variant": variant,
            "input": input_text[:100],
            "output": output[:200],
            "correct": None if expected is None else (output.strip().lower() == str(expected).strip().lower()),
            "user_feedback": user_feedback
        })
    
    def analyze(self) -> dict:
        """Analyzes the accumulated results."""
        from collections import defaultdict
        
        by_variant = defaultdict(list)
        for r in self.results:
            if r["correct"] is not None:
                by_variant[r["variant"]].append(r["correct"])
        
        if not by_variant:
            return {"error": "No results to analyze"}
        
        stats_by_variant = {}
        for variant, correct in by_variant.items():
            stats_by_variant[variant] = {
                "n": len(correct),
                "accuracy": sum(correct) / len(correct)
            }
        
        # Statistical test if there's data from both variants
        summary = {"variants": stats_by_variant}
        
        if "A" in by_variant and "B" in by_variant:
            _, p_value = stats.ttest_ind(by_variant["A"], by_variant["B"])
            summary["p_value"] = p_value
            summary["significant"] = p_value < 0.05
            
            acc_a = stats_by_variant["A"]["accuracy"]
            acc_b = stats_by_variant["B"]["accuracy"]
            summary["winner"] = "B" if acc_b > acc_a and p_value < 0.05 else ("A" if acc_a > acc_b and p_value < 0.05 else "TIE")
        
        return summary
    
    def should_stop(self, max_n: int = 1000) -> bool:
        """Checks whether the test should stop (enough N reached)."""
        total_n = len(self.results)
        return total_n >= max_n


# Usage in a FastAPI API:
"""
router = ABRouter(prompt_a=PROMPT_V1, prompt_b=PROMPT_V2, split=0.5)

@app.post("/classify")
async def classify(request: ClassifyRequest):
    prompt, variant = router.get_variant(user_id=request.user_id)
    
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt.format(input=request.text)}],
        temperature=0
    )
    output = response.choices[0].message.content
    
    router.log_result(variant, request.text, output)
    
    return {"result": output, "variant": variant}
"""

Interpreting the Results

def interpret_ab_test(result: dict) -> str:
    """
    Generates a natural-language interpretation of the A/B test.
    Useful for reporting to the team or in pull requests.
    """
    winner = result.get("winner", "INCONCLUSIVE")
    mean_a = result.get("mean_a", 0)
    mean_b = result.get("mean_b", 0)
    p_value = result.get("p_value")
    n_a = result.get("n_a", 0)
    n_b = result.get("n_b", 0)
    
    if winner == "TIE (not significant)":
        interpretation = f"""
## A/B Test Result

**Conclusion: There is no statistically significant difference**

- Prompt A accuracy: {mean_a:.2%} (n={n_a})
- Prompt B accuracy: {mean_b:.2%} (n={n_b})
- p-value: {p_value:.3f} (> 0.05 — not significant)

**Recommendation:** Keep Prompt A. Prompt B doesn't show enough improvement.
If you expect a larger improvement, consider growing the golden set.
"""
    elif winner == "B":
        delta = mean_b - mean_a
        interpretation = f"""
## A/B Test Result

**Conclusion: Prompt B is BETTER ✅**

- Prompt A accuracy: {mean_a:.2%} (n={n_a})
- Prompt B accuracy: {mean_b:.2%} (n={n_b})
- Improvement: {delta:+.2%} ({delta/mean_a*100:+.1f}% relative)
- p-value: {p_value:.3f} (< 0.05 — significant)

**Recommendation:** Ship Prompt B. The improvement is statistically significant.
"""
    elif winner == "A":
        delta = mean_b - mean_a
        interpretation = f"""
## A/B Test Result

**Conclusion: Prompt A is BETTER (Prompt B regressed) ⚠️**

- Prompt A accuracy: {mean_a:.2%} (n={n_a})
- Prompt B accuracy: {mean_b:.2%} (n={n_b})
- Change: {delta:+.2%}
- p-value: {p_value:.3f}

**Recommendation:** Don't deploy Prompt B. Review the changes you made.
"""
    else:
        interpretation = "## A/B Test Result\n**Inconclusive** — Not enough data."
    
    return interpretation.strip()


# Full usage example:
def full_ab_test_example():
    from openai import OpenAI
    client = OpenAI()
    
    # Minimal golden set for the demo
    golden_set = [
        {"id": "1", "input": "I love this product", "expected_output": "POSITIVE"},
        {"id": "2", "input": "Terrible, completely useless", "expected_output": "NEGATIVE"},
        {"id": "3", "input": "The package arrived today", "expected_output": "NEUTRAL"},
        # ... more examples in real production
    ]
    
    PROMPT_A = "Classify as POSITIVE, NEGATIVE or NEUTRAL: {input}. Only the category."
    
    PROMPT_B = """You are an expert sentiment classifier.
Classify the following text as POSITIVE, NEGATIVE, or NEUTRAL.
Consider the overall sentiment, not just individual words.
Answer ONLY with the category.

Text: {input}
Category:"""
    
    result = ab_test_offline(PROMPT_A, PROMPT_B, golden_set)
    summary = result.summary()
    
    print(interpret_ab_test(summary))
    
    return result

Common Mistakes in A/B Testing

Mistake 1: Peeking (Looking Too Early)

Problem: You run the test with 50 examples, see that B is winning,
         then stop the test and declare B the winner.
         
Why it's bad: With a small N, random variation can be large.
              You declare a winner when there isn't enough evidence yet.

Solution: Define N before the test and DON'T look until you reach it.

def test_with_fixed_n(prompt_a, prompt_b, golden_set, min_n=200):
    """
    Test that doesn't allow 'peeking' — it only reports once min_n is reached.
    """
    if len(golden_set) < min_n:
        raise ValueError(
            f"The golden set has {len(golden_set)} examples, "
            f"it needs at least {min_n} for this test. "
            f"Add more examples or lower your min_expected_improvement."
        )
    
    # Run with a fixed N
    result = ab_test_offline(prompt_a, prompt_b, golden_set, n_per_variant=min_n)
    return result

Mistake 2: Multiple Comparisons

Problem: You test prompts A, B, C, D, E... against the baseline.
         With 5 comparisons, the probability of a false positive goes up.

Bonferroni correction:
corrected_alpha = 0.05 / n_comparisons

def bonferroni_correction(
    results: list[dict],
    initial_alpha: float = 0.05
) -> list[dict]:
    """Applies the Bonferroni correction for multiple comparisons."""
    n = len(results)
    corrected_alpha = initial_alpha / n
    
    for r in results:
        if "p_value" in r:
            r["corrected_significant"] = r["p_value"] < corrected_alpha
            r["corrected_alpha"] = corrected_alpha
    
    return results

Mistake 3: Sample Contamination

Problem: You use the same examples to evaluate A and B, 
         and the results correlate artificially.
         
Solution: For small golden sets (< 200 examples), 
          using the same examples is acceptable but mention it.
          For large golden sets, split them randomly.

Troubleshooting

Problem 1: Non-significant result even though you expected an improvement

Symptom: p-value = 0.34, you can't conclude anything.

Most common cause: The golden set is too small.

Solution:

# Calculate how many more you need
r = calculate_sample_size(baseline_rate=0.91, min_expected_improvement=0.03)
print(f"You need {r['n_per_variant']} per variant")

# If you have 50 and need 350, the test is invalid — don't draw conclusions

Problem 2: A wins when you expected B

Symptom: The "improved" prompt B has lower accuracy.

Cause: The change introduced an unexpected regression.

Action:

# 1. Look at the cases where B failed but A didn't
def analyze_b_failures(prompt_a, prompt_b, golden_set):
    b_failures_a_passes = []
    
    for ex in golden_set:
        out_a = run_prompt(prompt_a, ex["input"])[0]
        out_b = run_prompt(prompt_b, ex["input"])[0]
        
        correct_a = out_a.lower() == str(ex["expected_output"]).lower()
        correct_b = out_b.lower() == str(ex["expected_output"]).lower()
        
        if correct_a and not correct_b:
            b_failures_a_passes.append({
                "id": ex["id"],
                "input": ex["input"],
                "expected": ex["expected_output"],
                "output_a": out_a,
                "output_b": out_b
            })
    
    return b_failures_a_passes

# Look for patterns in the failures → understand what prompt B broke

Problem 3: A tie across metrics (B wins accuracy, A wins cost)

Symptom: B has better accuracy (+3%) but also costs more (+20%).

Action: Decide according to business priorities:

def decide_winner_with_weights(
    metric_comparison: dict,
    weights: dict[str, float]
) -> str:
    """
    Determines the winner taking multiple weighted metrics into account.
    
    weights: {"accuracy": 0.6, "cost": 0.3, "latency": 0.1}
    """
    assert abs(sum(weights.values()) - 1.0) < 0.001
    
    score_a = 0.0
    score_b = 0.0
    
    for metric, weight in weights.items():
        if metric not in metric_comparison:
            continue
        
        m = metric_comparison[metric]
        if not m.get("significant"):
            # A tie, it doesn't count for either side
            continue
        
        winner = m.get("winner")
        if winner == "A":
            score_a += weight
        elif winner == "B":
            score_b += weight
    
    if score_b > score_a:
        return f"B (weighted score: {score_b:.2f} vs {score_a:.2f})"
    elif score_a > score_b:
        return f"A (weighted score: {score_a:.2f} vs {score_b:.2f})"
    else:
        return f"TIE ({score_a:.2f} each)"


# Example:
# accuracy is crucial, cost matters, latency is secondary
weights = {"accuracy": 0.6, "cost": 0.3, "latency": 0.1}
winner = decide_winner_with_weights(metric_comparison, weights)

Exercises

Exercise 1: Calculate the sample size for your case

For a classifier with a current accuracy of 88%, calculate how many examples you need to detect a 5% improvement with 80% power and 95% confidence.

from scipy import stats
import math

def calculate_sample_size(baseline_rate, improvement, alpha=0.05, power=0.80):
    p1 = baseline_rate
    p2 = baseline_rate + improvement
    
    z_alpha = stats.norm.ppf(1 - alpha / 2)
    z_beta = stats.norm.ppf(power)
    p_pool = (p1 + p2) / 2
    
    n = math.ceil(
        (z_alpha * math.sqrt(2 * p_pool * (1 - p_pool)) + 
         z_beta * math.sqrt(p1 * (1 - p1) + p2 * (1 - p2))) ** 2
        / (p2 - p1) ** 2
    )
    
    return n

n = calculate_sample_size(baseline_rate=0.88, improvement=0.05)
print(f"You need {n} examples PER variant ({n*2} total)")
# Typical result: ~248 per variant (496 total)

# For cost context:
cost_per_request = 0.001  # $0.001 per example (approximate with gpt-4o-mini)
total_cost = n * 2 * cost_per_request
print(f"Estimated test cost: ${total_cost:.2f}")

Exercise 2: A/B test with interpretation

Write the full code to compare two versions of a summarization prompt and report the winner with statistical justification.

from openai import OpenAI
from scipy import stats

client = OpenAI()

PROMPT_A = "Summarize in 1 sentence: {input}"

PROMPT_B = """Summarize the following text in exactly one sentence.
Include the most important point.
Use no more than 30 words.

Text: {input}
Summary:"""

texts = [
    "The central bank announced today a 0.25-point hike in interest rates, the third in a row this year, aiming to fight inflation that reached 8.2% last quarter.",
    "Apple unveiled the new iPhone 17 with significant camera improvements, including a 200-megapixel sensor and 8K recording, available next month with prices starting at $999.",
]

answers_a = []
answers_b = []

for text in texts:
    resp_a = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": PROMPT_A.format(input=text)}],
        temperature=0
    ).choices[0].message.content.strip()
    
    resp_b = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": PROMPT_B.format(input=text)}],
        temperature=0
    ).choices[0].message.content.strip()
    
    answers_a.append(resp_a)
    answers_b.append(resp_b)
    
    print(f"A: {resp_a[:80]}...")
    print(f"B: {resp_b[:80]}...")
    print()

# For evaluation: measure length as a proxy (shorter = more concise)
lengths_a = [len(r.split()) for r in answers_a]
lengths_b = [len(r.split()) for r in answers_b]

print(f"Average words A: {sum(lengths_a)/len(lengths_a):.1f}")
print(f"Average words B: {sum(lengths_b)/len(lengths_b):.1f}")
print("(Demo with a small n — in production use a golden set with N>=200)")

Exercise 3: Spot the peeking problem

Explain with code why the following A/B test has a peeking problem and how to fix it:

# Code with the problem
for i, example in enumerate(golden_set):
    # evaluate A and B...
    if i == 20:  # We check with 20 examples
        if mean_b > mean_a + 0.05:
            print("B wins! Deploy.")
            break

# The problem: with 20 examples, the p-value is usually > 0.05
# A 5% difference can be pure statistical noise
# By stopping early, the "winner" can be a false positive

# THE CORRECT SOLUTION:

from scipy import stats

def correct_ab_test(prompt_a, prompt_b, golden_set, min_n=200):
    """Test without peeking — a fixed N decided BEFORE the test."""
    
    # 1. Calculate the required N BEFORE starting
    required_n = calculate_sample_size(baseline_rate=0.90, improvement=0.03)
    
    if len(golden_set) < required_n:
        raise ValueError(
            f"Insufficient golden set: it has {len(golden_set)}, it needs {required_n}. "
            "Add more examples or accept lower precision."
        )
    
    # 2. Run ALL the examples without looking at intermediate results
    scores_a, scores_b = [], []
    
    import random
    sample = random.sample(golden_set, min(required_n * 2, len(golden_set)))
    
    for ex in sample[:required_n]:  # Prompt A
        out = run_prompt(prompt_a, ex["input"])[0]
        scores_a.append(1.0 if out.lower() == str(ex["expected_output"]).lower() else 0.0)
    
    for ex in sample[required_n:2*required_n]:  # Prompt B
        out = run_prompt(prompt_b, ex["input"])[0]
        scores_b.append(1.0 if out.lower() == str(ex["expected_output"]).lower() else 0.0)
    
    # 3. Evaluate ONLY ONCE, at the end
    mean_a = sum(scores_a) / len(scores_a)
    mean_b = sum(scores_b) / len(scores_b)
    _, p_value = stats.ttest_ind(scores_a, scores_b)
    
    winner = "B" if mean_b > mean_a and p_value < 0.05 else ("A" if mean_a > mean_b and p_value < 0.05 else "TIE")
    
    print(f"A: {mean_a:.2%}, B: {mean_b:.2%}, p={p_value:.3f}, Winner: {winner}")
    return winner

Summary

• A/B testing: The rigorous method for comparing two prompt versions with statistical certainty
• Sample size: Calculate it BEFORE the test — with an insufficient N, the results are invalid
• p-value < 0.05: The standard threshold for statistical significance
• Effect size: Not just "is it significant" but "how much does it improve?"
• Multiple metrics: Accuracy + latency + cost — decide with weights based on priorities
• Peeking: Don't stop the test before reaching the minimum N
• Online A/B: ABRouter for testing in production with real users

Additional resources

1. Evan Miller A/B Test Calculator — Sample size calculator
2. scipy.stats — Statistical tests in Python
3. A/B Testing Statistics Guide — Comprehensive guide
4. Stats for A/B Testing — Common mistakes
5. Statsmodels — Advanced statistics library for Python