2. Mocking LLM Responses

Description

Correctly mocking the LLM's responses is the foundation of unit testing for AI. This session covers unittest.mock.patch, MagicMock, AsyncMock, and how to structure mocks that simulate the real OpenAI/Anthropic API. Realistic mocks avoid surprises when the code interacts with the real API and ensure that the tests validate the correct behavior.

Why mocks must be realistic

A poorly built mock is worse than no mock: it gives you false confidence. Consider this example:

# Trivial mock (do NOT do this)
@pytest.fixture
def bad_mock():
    client = MagicMock()
    client.chat.completions.create.return_value = "Hello"  # ❌ Simple string
    return client

def test_with_bad_mock(bad_mock):
    result = analyze_sentiment("text", client=bad_mock)
    # ⚠️ If your code does: response.choices[0].message.content
    # it will fail with AttributeError: 'str' object has no attribute 'choices'
    # — BUT in this test it may pass if the test doesn't use choices either

# Realistic mock (DO do this)
@pytest.fixture
def good_mock():
    client = MagicMock()
    # Replicate the exact structure of openai.ChatCompletion
    mock_choice = MagicMock()
    mock_choice.message.content = '{"sentiment": "positive", "score": 0.85}'
    mock_choice.message.role = "assistant"
    mock_choice.finish_reason = "stop"
    mock_choice.index = 0

    mock_response = MagicMock()
    mock_response.choices = [mock_choice]
    mock_response.usage.prompt_tokens = 45
    mock_response.usage.completion_tokens = 25
    mock_response.usage.total_tokens = 70
    mock_response.model = "gpt-4o-mini"
    mock_response.id = "chatcmpl-mock-test-123"
    mock_response.created = 1700000000

    client.chat.completions.create.return_value = mock_response
    return client

Golden rule: Your mock must have exactly the same structure as the real object. That way, if your code accesses response.usage.total_tokens, the mock doesn't fail.

OpenAI response structure: complete reference

To build realistic mocks, you need to know the real structure:

# What client.chat.completions.create() really returns
# (Simplified but faithful to the real structure)
{
    "id": "chatcmpl-abc123",
    "object": "chat.completion",
    "created": 1700000000,
    "model": "gpt-4o-mini",
    "choices": [
        {
            "index": 0,
            "message": {
                "role": "assistant",
                "content": '{"sentiment": "positive", "score": 0.85}',
                "tool_calls": None,
                "function_call": None
            },
            "finish_reason": "stop",   # or "length", "content_filter", "tool_calls"
            "logprobs": None
        }
    ],
    "usage": {
        "prompt_tokens": 45,
        "completion_tokens": 25,
        "total_tokens": 70,
        "completion_tokens_details": {
            "reasoning_tokens": 0,
            "audio_tokens": 0
        }
    },
    "system_fingerprint": "fp_abc123"
}

Helper function: create_openai_chat_response

This function is the standard you'll use in all the module's tests:

# tests/helpers.py
from unittest.mock import MagicMock

def create_openai_chat_response(
    content: str,
    model: str = "gpt-4o-mini",
    prompt_tokens: int = 45,
    completion_tokens: int = 25,
    finish_reason: str = "stop"
) -> MagicMock:
    """
    Creates a mock that replicates the real structure of openai.ChatCompletion.

    Usage:
        mock_response = create_openai_chat_response('{"sentiment": "positive"}')
        client.chat.completions.create.return_value = mock_response
    """
    # Build the choice
    mock_message = MagicMock()
    mock_message.role = "assistant"
    mock_message.content = content
    mock_message.tool_calls = None
    mock_message.function_call = None

    mock_choice = MagicMock()
    mock_choice.index = 0
    mock_choice.message = mock_message
    mock_choice.finish_reason = finish_reason
    mock_choice.logprobs = None

    # Build usage
    mock_usage = MagicMock()
    mock_usage.prompt_tokens = prompt_tokens
    mock_usage.completion_tokens = completion_tokens
    mock_usage.total_tokens = prompt_tokens + completion_tokens

    # Build the complete response
    mock_response = MagicMock()
    mock_response.id = "chatcmpl-mock-test-abc123"
    mock_response.object = "chat.completion"
    mock_response.created = 1700000000
    mock_response.model = model
    mock_response.choices = [mock_choice]
    mock_response.usage = mock_usage
    mock_response.system_fingerprint = "fp_test"

    return mock_response

The three ways to patch

Way 1: @patch decorator

from unittest.mock import patch, MagicMock
from tests.helpers import create_openai_chat_response

@patch("app.sentiment.client.chat.completions.create")
def test_analyze_sentiment_decorator(mock_create):
    """Using @patch as a decorator."""
    # Configure the mock
    mock_create.return_value = create_openai_chat_response(
        '{"sentiment": "positive", "score": 0.9, "explanation": "Positive text"}'
    )

    # Run
    result = analyze_sentiment("I love this product")

    # Assertions about the result
    assert result["sentiment"] == "positive"
    assert result["score"] == 0.9

    # Assertions about the mock (verify it was called correctly)
    mock_create.assert_called_once()
    call_args = mock_create.call_args
    assert call_args.kwargs["model"] == "gpt-4o-mini"
    assert "I love this product" in str(call_args.kwargs["messages"])

Way 2: with patch context manager

def test_analyze_sentiment_context():
    """Using with patch as a context manager."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.return_value = create_openai_chat_response(
            '{"sentiment": "negative", "score": 0.2, "explanation": "Negative text"}'
        )

        result = analyze_sentiment("This product is terrible")

        assert result["sentiment"] == "negative"
        assert result["score"] == 0.2

Way 3: Fixture in conftest.py (the most used)

# tests/conftest.py
import pytest
from unittest.mock import MagicMock
from tests.helpers import create_openai_chat_response

@pytest.fixture
def mock_openai_client():
    """Standard fixture: OpenAI client mock for unit tests."""
    client = MagicMock()
    client.chat.completions.create.return_value = create_openai_chat_response(
        '{"sentiment": "neutral", "score": 0.5, "explanation": "Neutral text"}'
    )
    return client

# In the test:
def test_analyze_sentiment_fixture(mock_openai_client):
    result = analyze_sentiment("Test text", client=mock_openai_client)
    assert result["sentiment"] == "neutral"

Comparison of the three approaches

The "where it's used" pattern (critical)

The most common mocking error: patching in the wrong place.

# ❌ INCORRECT: patch where it's DEFINED
@patch("openai.OpenAI")
def test_wrong(mock_openai):
    result = analyze_sentiment("text")  # Doesn't work — the object was already imported

# ✅ CORRECT: patch where it's USED (where it's imported in your module)
@patch("app.sentiment.openai.OpenAI")    # if app.sentiment does: import openai
# or
@patch("app.sentiment.OpenAI")           # if app.sentiment does: from openai import OpenAI
# or
@patch("app.sentiment.client")           # if app.sentiment has: client = OpenAI(...)

The rule: patch("module.where.the.name.is.used")

# app/sentiment.py
from openai import OpenAI          # → patch("app.sentiment.OpenAI")
import openai                       # → patch("app.sentiment.openai.OpenAI")
client = OpenAI()                   # → patch("app.sentiment.client")

Mocking errors and edge cases

Errors are as important as success cases. Your app must handle LLM errors gracefully.

Simulating API errors

import openai
from unittest.mock import patch

def test_handles_rate_limit():
    """The app must handle a rate limit without crashing."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        # Simulate a 429 error
        mock_create.side_effect = openai.RateLimitError(
            message="Rate limit exceeded",
            response=MagicMock(status_code=429),
            body={"error": {"message": "Rate limit exceeded"}}
        )

        result = analyze_sentiment("text")

        # The app must return a controlled error, not crash
        assert result["error"] == "rate_limit"
        # or: assert result is None
        # or: verify it logs the error

def test_handles_timeout():
    """The app must handle timeouts."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.side_effect = openai.APITimeoutError(
            request=MagicMock()
        )

        with pytest.raises(TimeoutError):
            analyze_sentiment("text")
        # or verify the fallback behavior

def test_handles_api_error():
    """Generic API error."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.side_effect = openai.APIError(
            message="Service unavailable",
            request=MagicMock(),
            body=None
        )

        result = analyze_sentiment("text")
        assert "error" in result

Edge cases in the content

@pytest.fixture
def mock_client_empty_response():
    """Mock that returns an empty response."""
    client = MagicMock()
    client.chat.completions.create.return_value = create_openai_chat_response("")
    return client

@pytest.fixture
def mock_client_malformed_json():
    """Mock that returns malformed JSON."""
    client = MagicMock()
    client.chat.completions.create.return_value = create_openai_chat_response(
        '{"sentiment": "positive", "score": 0.9'  # Incomplete JSON
    )
    return client

@pytest.fixture
def mock_client_markdown_json():
    """Mock that returns JSON inside a markdown code block."""
    client = MagicMock()
    client.chat.completions.create.return_value = create_openai_chat_response(
        '```json\n{"sentiment": "positive", "score": 0.9}\n```'
    )
    return client

@pytest.fixture
def mock_client_truncated():
    """Mock that simulates a truncated response (finish_reason=length)."""
    client = MagicMock()
    client.chat.completions.create.return_value = create_openai_chat_response(
        '{"sentiment": "positive", "score": 0.9, "explanation": "The text is very',  # Truncated
        finish_reason="length"
    )
    return client

# Tests using these mocks:
def test_handles_empty_response(mock_client_empty_response):
    """The parser must handle an empty response without crashing."""
    result = analyze_sentiment("text", client=mock_client_empty_response)
    assert result["sentiment"] == "unknown"  # or the fallback you define

def test_handles_malformed_json(mock_client_malformed_json):
    """The parser must handle malformed JSON."""
    result = analyze_sentiment("text", client=mock_client_malformed_json)
    assert "error" in result or result.get("sentiment") == "unknown"

def test_handles_markdown_json(mock_client_markdown_json):
    """The parser must extract JSON from markdown code blocks."""
    result = analyze_sentiment("text", client=mock_client_markdown_json)
    assert result["sentiment"] == "positive"  # Must parse correctly

Mock with side_effect for multiple calls

def test_retry_logic():
    """The app retries in case of a transient error."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        # First call fails, second succeeds
        mock_create.side_effect = [
            openai.APITimeoutError(request=MagicMock()),  # First: timeout
            create_openai_chat_response(                    # Second: success
                '{"sentiment": "positive", "score": 0.9}'
            )
        ]

        result = analyze_sentiment("text")

        assert result["sentiment"] == "positive"
        assert mock_create.call_count == 2  # It was called twice

def test_chain_multiple_calls():
    """A chain makes two LLM calls: extract + classify."""
    with patch("app.chain.client.chat.completions.create") as mock_create:
        mock_create.side_effect = [
            create_openai_chat_response('["feature1", "feature2"]'),  # Extraction
            create_openai_chat_response('{"category": "tech", "confidence": 0.95}')  # Classification
        ]

        result = extract_and_classify("Input text")

        assert result["category"] == "tech"
        assert mock_create.call_count == 2

Async mocking with AsyncMock

If your app uses async LLM calls, you need AsyncMock:

# app/async_sentiment.py
import asyncio
import openai

async def analyze_sentiment_async(text: str, client) -> dict:
    response = await client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": text}]
    )
    return parse_sentiment(response.choices[0].message.content)

# test_async.py
import pytest
from unittest.mock import AsyncMock, MagicMock
from tests.helpers import create_openai_chat_response

@pytest.mark.asyncio
async def test_analyze_sentiment_async():
    """Test of an async function with AsyncMock."""
    # AsyncMock for async functions
    mock_create = AsyncMock(
        return_value=create_openai_chat_response(
            '{"sentiment": "positive", "score": 0.9}'
        )
    )

    mock_client = MagicMock()
    mock_client.chat.completions.create = mock_create

    result = await analyze_sentiment_async("text", client=mock_client)

    assert result["sentiment"] == "positive"
    mock_create.assert_called_once()

# Async fixture:
@pytest.fixture
def async_mock_client():
    """Async mock of the OpenAI client."""
    client = MagicMock()
    client.chat.completions.create = AsyncMock(
        return_value=create_openai_chat_response(
            '{"sentiment": "neutral", "score": 0.5}'
        )
    )
    return client

Verify that the mock was called correctly

The output isn't the only thing that matters — you must also verify that your code called the LLM with the correct parameters:

def test_calls_llm_with_correct_params(mock_openai_client):
    """Verifies that the LLM is called with the correct parameters."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.return_value = create_openai_chat_response(
            '{"sentiment": "positive", "score": 0.9}'
        )

        analyze_sentiment("Test text")

        # Verify it was called exactly once
        mock_create.assert_called_once()

        # Get the arguments it was called with
        call_kwargs = mock_create.call_args.kwargs

        # Verify the model
        assert call_kwargs.get("model") == "gpt-4o-mini"

        # Verify that the text appeared in the messages
        messages = call_kwargs.get("messages", [])
        user_message = next(m for m in messages if m["role"] == "user")
        assert "Test text" in user_message["content"]

        # Verify the temperature (if you control it)
        assert call_kwargs.get("temperature", 0) == 0

def test_does_not_call_llm_for_cached_result(mock_openai_client):
    """Verifies that the cache avoids extra LLM calls."""
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.return_value = create_openai_chat_response(
            '{"sentiment": "positive", "score": 0.9}'
        )

        # First call
        analyze_sentiment("same text")
        # Second call (must use the cache)
        analyze_sentiment("same text")

        # It must have called the LLM only once
        mock_create.assert_called_once()

Comparison: unittest.mock vs pytest-mock

# With unittest.mock (standard Python):
from unittest.mock import patch, MagicMock

@patch("app.sentiment.client.chat.completions.create")
def test_with_unittest(mock_create):
    mock_create.return_value = create_openai_chat_response("...")
    # ...

# With pytest-mock (more ergonomic):
def test_with_pytest_mock(mocker):
    mock_create = mocker.patch("app.sentiment.client.chat.completions.create")
    mock_create.return_value = create_openai_chat_response("...")
    # ...

Anthropic structure: differences

If you use Anthropic, the mock structure is different:

# Real Anthropic response:
{
    "id": "msg_abc123",
    "type": "message",
    "role": "assistant",
    "content": [
        {
            "type": "text",
            "text": '{"sentiment": "positive", "score": 0.9}'
        }
    ],
    "model": "claude-3-haiku-20240307",
    "stop_reason": "end_turn",
    "usage": {
        "input_tokens": 45,
        "output_tokens": 25
    }
}

# Mock for Anthropic:
def create_anthropic_response(content: str, model: str = "claude-3-haiku-20240307"):
    mock_content_block = MagicMock()
    mock_content_block.type = "text"
    mock_content_block.text = content

    mock_usage = MagicMock()
    mock_usage.input_tokens = 45
    mock_usage.output_tokens = 25

    mock_response = MagicMock()
    mock_response.id = "msg_mock_test"
    mock_response.type = "message"
    mock_response.role = "assistant"
    mock_response.content = [mock_content_block]
    mock_response.model = model
    mock_response.stop_reason = "end_turn"
    mock_response.usage = mock_usage

    return mock_response

Abstraction tip: If your app can use OpenAI or Anthropic, consider a normalize_response(response, provider) function in your code that unifies the interface — that also makes testing simpler.

Exercises

Exercise 1: Build a realistic mock

Your generate_title function calls OpenAI and parses the output like this:

raw = response.choices[0].message.content
return raw.strip()  # The title is the full text

Build a mock to test that:

1. The returned title is the mock's content
2. The function was called exactly once
3. The function was called with max_tokens=50

from unittest.mock import patch, MagicMock
from tests.helpers import create_openai_chat_response

def test_generate_title():
    with patch("app.title.client.chat.completions.create") as mock_create:
        mock_create.return_value = create_openai_chat_response(
            "  Introduction to Python  "  # With spaces to verify strip()
        )

        result = generate_title("Write a Python tutorial")

        # Verify the result
        assert result == "Introduction to Python"  # strip() applied

        # Verify the call
        mock_create.assert_called_once()
        call_kwargs = mock_create.call_args.kwargs
        assert call_kwargs.get("max_tokens") == 50

Exercise 2: Error mock with a fallback

Your app has this behavior: if the LLM fails, it returns {"sentiment": "unknown", "error": True}.

Write the test that verifies this behavior:

import openai
from unittest.mock import patch, MagicMock

def test_fallback_on_api_error():
    with patch("app.sentiment.client.chat.completions.create") as mock_create:
        mock_create.side_effect = openai.APIConnectionError(
            message="Connection failed",
            request=MagicMock()
        )

        result = analyze_sentiment("Test text")

        # Verify the fallback
        assert result["sentiment"] == "unknown"
        assert result["error"] is True
        # The app must not re-raise the exception

Exercise 3: Multiple sequential calls

Your summarize_and_classify function makes two calls:

1. First: summarizes the text
2. Second: classifies the summary

Write the test with side_effect that verifies both calls happen:

from tests.helpers import create_openai_chat_response

def test_summarize_and_classify():
    with patch("app.chain.client.chat.completions.create") as mock_create:
        # Responses for the first and second call
        mock_create.side_effect = [
            create_openai_chat_response(
                '{"summary": "Python is popular for AI"}'
            ),
            create_openai_chat_response(
                '{"category": "technology", "confidence": 0.95}'
            )
        ]

        result = summarize_and_classify("Long text about Python...")

        # Verify the final result
        assert result["summary"] == "Python is popular for AI"
        assert result["category"] == "technology"

        # Verify exactly 2 calls were made
        assert mock_create.call_count == 2

Exercise 4: AsyncMock for an async function

Convert the Exercise 1 test to work with an async version of generate_title:

# The async function:
async def generate_title_async(prompt: str, client) -> str:
    response = await client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt}],
        max_tokens=50
    )
    return response.choices[0].message.content.strip()

import pytest
from unittest.mock import AsyncMock, MagicMock
from tests.helpers import create_openai_chat_response

@pytest.mark.asyncio
async def test_generate_title_async():
    mock_client = MagicMock()
    mock_client.chat.completions.create = AsyncMock(
        return_value=create_openai_chat_response("  Introduction to Python  ")
    )

    result = await generate_title_async("Write a Python tutorial", client=mock_client)

    assert result == "Introduction to Python"
    mock_client.chat.completions.create.assert_called_once()
    call_kwargs = mock_client.chat.completions.create.call_args.kwargs
    assert call_kwargs.get("max_tokens") == 50

Exercise 5: Diagnose the patch

The following test doesn't work — the mock isn't applied. Why, and how do you fix it?

# app/summarizer.py
import openai
client = openai.OpenAI()

def summarize(text: str) -> str:
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": text}]
    )
    return response.choices[0].message.content

# test_summarizer.py
@patch("openai.OpenAI")   # ← Is this right?
def test_summarize(mock_openai_class):
    mock_instance = MagicMock()
    mock_openai_class.return_value = mock_instance
    mock_instance.chat.completions.create.return_value = create_openai_chat_response("Test")

    result = summarize("text")  # ← Does it work?

@patch("app.summarizer.client")
def test_summarize(mock_client):
    mock_client.chat.completions.create.return_value = create_openai_chat_response("Test")

    result = summarize("text")

    assert result == "Test"
    mock_client.chat.completions.create.assert_called_once()

def summarize(text: str, client=None) -> str:
    if client is None:
        client = openai.OpenAI()
    response = client.chat.completions.create(...)
    return response.choices[0].message.content

Summary

• Mocks must replicate the real structure of the API (choices, message, usage): use create_openai_chat_response
• Patch where it's used, not where it's defined — patch("module.that.uses.it.name")
• Three ways to patch: decorator, context manager, fixture — each has its place
• Edge cases: empty response, malformed JSON, finish_reason=length, API errors
• AsyncMock for async functions — don't use MagicMock for coroutines
• Verify interactions: assert_called_once(), call_args.kwargs to validate that the LLM is called correctly

Additional resources

1. unittest.mock — Official Python documentation — Complete reference
2. Where to patch — Python docs — Critical guide for understanding the patch scope
3. pytest-mock — Plugin for cleaner syntax
4. OpenAI API Reference — Chat Completions — Real structure of the response object
5. Anthropic API Reference — Structure of Anthropic messages
6. Testing async code with pytest-asyncio — For async functions
7. Mock cookbook — Quick guide with common patterns