5. Testing Parsers and Output Processors

Description

Parsers and output processors are the easiest and most valuable logic to test in an AI app: they're 100% deterministic, need no mocks, and represent most of the code that can break when the LLM produces variations. This capsule covers how to test functions that transform the LLM's raw output into structured data. It's the "boring testing" that most people ignore — and where the ROI is highest.

Why are parsers so important?

The LLM can produce semantically "correct" output but with format variations that break your app:

# The LLM must return JSON but sometimes produces:
'{"sentiment": "positive"}'                           # Ideal format
'```json\n{"sentiment": "positive"}\n```'             # With markdown
'The result is: {"sentiment": "positive"}'             # With preceding text
'{"sentiment": "positive",}'                           # Trailing comma (invalid JSON)
'{"sentiment":"positive"}'                             # No spaces
'{"Sentiment": "Positive"}'                            # Different capitalization
'\n\n{"sentiment": "positive"}\n\n'                    # With extra whitespace

Without a robust parser, any of these formats can break your app. With parser tests, you can verify that you handle all of them correctly.

What is a parser in an AI context

A parser takes the LLM's raw output (a string) and converts it into structured data:

# Input: raw string from the LLM (may have markdown, extra text, variations)
# Output: data structure (dict, list, Pydantic model)

def parse_sentiment_response(raw: str) -> dict:
    """
    Extracts and parses the JSON from the LLM response.
    
    Handles multiple formats the LLM may produce:
    - Direct JSON: '{"sentiment": "positive"}'
    - JSON in markdown: '```json\\n{...}\\n```'
    - JSON with preceding text: 'The analysis: {...}'
    """
    if not raw or not raw.strip():
        raise ValueError("The LLM response is empty")
    
    # Try parsing directly first
    try:
        return json.loads(raw.strip())
    except json.JSONDecodeError:
        pass
    
    # Look for JSON in markdown code blocks
    markdown_pattern = r'```(?:json)?\s*\n?(.*?)\n?```'
    match = re.search(markdown_pattern, raw, re.DOTALL)
    if match:
        try:
            return json.loads(match.group(1).strip())
        except json.JSONDecodeError:
            pass
    
    # Look for any JSON object in the text
    json_pattern = r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}'
    matches = re.findall(json_pattern, raw, re.DOTALL)
    for match in matches:
        try:
            return json.loads(match)
        except json.JSONDecodeError:
            continue
    
    raise ValueError(f"No valid JSON found in the response: {raw[:100]!r}")

Test structure for parsers

The optimal structure is parametrize with multiple input formats:

import pytest
import json

# Test cases for the parser
VALID_PARSER_CASES = [
    pytest.param(
        '{"sentiment": "positive", "score": 0.9}',
        {"sentiment": "positive", "score": 0.9},
        id="json_direct"
    ),
    pytest.param(
        '```json\n{"sentiment": "positive", "score": 0.9}\n```',
        {"sentiment": "positive", "score": 0.9},
        id="json_in_markdown"
    ),
    pytest.param(
        '```\n{"sentiment": "positive", "score": 0.9}\n```',
        {"sentiment": "positive", "score": 0.9},
        id="json_in_code_block_no_language"
    ),
    pytest.param(
        'The sentiment analysis is: {"sentiment": "positive", "score": 0.9}',
        {"sentiment": "positive", "score": 0.9},
        id="json_with_preceding_text"
    ),
    pytest.param(
        '\n\n{"sentiment": "positive", "score": 0.9}\n\n',
        {"sentiment": "positive", "score": 0.9},
        id="json_with_whitespace"
    ),
    pytest.param(
        '{"sentiment": "positive", "score": 0.9, "extra_field": "ignored"}',
        {"sentiment": "positive", "score": 0.9, "extra_field": "ignored"},
        id="json_with_extra_fields"
    ),
    pytest.param(
        '{"sentiment": "positive", "score": 0.9, "keywords": ["good", "excellent"]}',
        {"sentiment": "positive", "score": 0.9, "keywords": ["good", "excellent"]},
        id="json_with_array"
    ),
]

@pytest.mark.parametrize("raw_input,expected", VALID_PARSER_CASES)
def test_parse_sentiment_valid_formats(raw_input, expected):
    """The parser handles all the valid formats the LLM can produce."""
    result = parse_sentiment_response(raw_input)
    assert result == expected

Edge case and error tests

Edge cases are where parsers fail most in production:

ERROR_CASES = [
    pytest.param("", ValueError, id="empty_input"),
    pytest.param("   ", ValueError, id="whitespace_only_input"),
    pytest.param("Text with no JSON", ValueError, id="no_json"),
    pytest.param("No structure here", ValueError, id="free_text"),
]

@pytest.mark.parametrize("raw_input,expected_exception", ERROR_CASES)
def test_parse_sentiment_invalid_inputs(raw_input, expected_exception):
    """The parser raises appropriate exceptions for invalid inputs."""
    with pytest.raises(expected_exception):
        parse_sentiment_response(raw_input)

def test_parse_malformed_json_raises():
    """Malformed JSON raises ValueError."""
    with pytest.raises((json.JSONDecodeError, ValueError)):
        parse_sentiment_response('{"sentiment": "positive", "score": 0.9')  # Not closed

def test_parse_truncated_response():
    """Truncated response (LLM cut off by token limit) raises an appropriate error."""
    truncated = '{"sentiment": "positive", "score": 0.9, "explanation": "The text is very'
    with pytest.raises((json.JSONDecodeError, ValueError)):
        parse_sentiment_response(truncated)

def test_parse_handles_unicode():
    """The parser handles Unicode characters correctly."""
    raw = '{"sentiment": "positivo", "keywords": ["excelente", "fantástico", "müde"]}'
    result = parse_sentiment_response(raw)
    assert "fantástico" in result["keywords"]
    assert "müde" in result["keywords"]

def test_parse_handles_nested_quotes():
    """The parser handles quotes inside the JSON."""
    raw = '{"sentiment": "positive", "explanation": "The product is \\"incredible\\""}'
    result = parse_sentiment_response(raw)
    assert result["explanation"] == 'The product is "incredible"'

Output processors: post-processing

An output processor takes the parser's dict and normalizes, validates, and structures it:

# app/processors.py

def process_sentiment_output(raw_dict: dict) -> dict:
    """
    Normalizes and validates the LLM's parsed output.
    
    Output guarantees:
    - sentiment: always one of ["positive", "negative", "neutral"]
    - score: always a float between 0.0 and 1.0
    - keywords: always a list (may be empty)
    - explanation: always a string (may be empty)
    """
    VALID_SENTIMENTS = {"positive", "negative", "neutral"}
    
    # Normalize sentiment: case-insensitive, with fallback
    raw_sentiment = raw_dict.get("sentiment", "").strip().lower()
    sentiment = raw_sentiment if raw_sentiment in VALID_SENTIMENTS else "neutral"
    
    # Clamp score to the range [0, 1]
    try:
        score = float(raw_dict.get("score", 0.5))
        score = max(0.0, min(1.0, score))
    except (ValueError, TypeError):
        score = 0.5
    
    # Keywords: convert to a list if a string, filter out empties
    raw_keywords = raw_dict.get("keywords", [])
    if isinstance(raw_keywords, str):
        keywords = [k.strip() for k in raw_keywords.split(",") if k.strip()]
    elif isinstance(raw_keywords, list):
        keywords = [str(k).strip() for k in raw_keywords if k and str(k).strip()]
    else:
        keywords = []
    
    # Explanation: string with strip and truncation
    explanation = str(raw_dict.get("explanation", "")).strip()[:500]
    
    return {
        "sentiment": sentiment,
        "score": score,
        "keywords": keywords,
        "explanation": explanation
    }

# Output processor tests:

def test_process_normalizes_sentiment():
    """The processor normalizes sentiment to lowercase."""
    assert process_sentiment_output({"sentiment": "POSITIVE", "score": 0.9})["sentiment"] == "positive"
    assert process_sentiment_output({"sentiment": "Negative", "score": 0.1})["sentiment"] == "negative"
    assert process_sentiment_output({"sentiment": "NEUTRAL", "score": 0.5})["sentiment"] == "neutral"

def test_process_invalid_sentiment_defaults_to_neutral():
    """Unrecognized sentiments map to neutral."""
    assert process_sentiment_output({"sentiment": "very_positive", "score": 0.9})["sentiment"] == "neutral"
    assert process_sentiment_output({"sentiment": "", "score": 0.5})["sentiment"] == "neutral"
    assert process_sentiment_output({})["sentiment"] == "neutral"

def test_process_clamps_score():
    """Out-of-range score is clamped to [0, 1]."""
    assert process_sentiment_output({"sentiment": "positive", "score": 1.5})["score"] == 1.0
    assert process_sentiment_output({"sentiment": "negative", "score": -0.1})["score"] == 0.0
    assert process_sentiment_output({"sentiment": "neutral", "score": 999})["score"] == 1.0

def test_process_handles_missing_score():
    """Missing score uses the default value (0.5)."""
    result = process_sentiment_output({"sentiment": "positive"})
    assert result["score"] == 0.5

def test_process_handles_invalid_score_type():
    """Score with an invalid type uses the default value."""
    assert process_sentiment_output({"sentiment": "positive", "score": "not-a-float"})["score"] == 0.5
    assert process_sentiment_output({"sentiment": "positive", "score": None})["score"] == 0.5

def test_process_normalizes_keywords():
    """Keywords are normalized to a clean list."""
    # Normal list
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9, "keywords": ["good", "  excellent  "]})
    assert result["keywords"] == ["good", "excellent"]  # Whitespace removed
    
    # Comma-separated string (the LLM sometimes does this)
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9, "keywords": "good, excellent, incredible"})
    assert result["keywords"] == ["good", "excellent", "incredible"]
    
    # Empty list
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9, "keywords": []})
    assert result["keywords"] == []
    
    # Missing
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9})
    assert result["keywords"] == []

def test_process_truncates_long_explanation():
    """A very long explanation is truncated to 500 characters."""
    long_explanation = "text " * 200  # 1000 characters
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9, "explanation": long_explanation})
    assert len(result["explanation"]) <= 500

Testing the complete pipeline: parser + processor

The complete pipeline is: raw string → parser → processor → final output. You can test each step in isolation AND the complete pipeline:

def process_llm_response(raw_string: str) -> dict:
    """Complete pipeline: parse + process."""
    parsed = parse_sentiment_response(raw_string)
    return process_sentiment_output(parsed)

# Test of the complete pipeline (no mock needed):
@pytest.mark.parametrize("raw_input,expected_sentiment,expected_score_range", [
    ('{"sentiment": "POSITIVE", "score": 0.9}', "positive", (0.8, 1.0)),
    ('```json\n{"sentiment": "negative", "score": 0.2}\n```', "negative", (0.0, 0.3)),
    ('{"sentiment": "invalid", "score": 5.0}', "neutral", (1.0, 1.0)),  # Clamped score
    ('The analysis: {"sentiment": "neutral", "score": 0.5}', "neutral", (0.4, 0.6)),
])
def test_full_pipeline(raw_input, expected_sentiment, expected_score_range):
    """Complete pipeline: parser + processor for multiple formats."""
    result = process_llm_response(raw_input)
    
    assert result["sentiment"] == expected_sentiment
    assert expected_score_range[0] <= result["score"] <= expected_score_range[1]

Using real LLM outputs as test cases

The most effective way to improve your parser tests: capture real LLM outputs and use them as cases:

# In development: log the raw output before parsing
import logging

logger = logging.getLogger(__name__)

def analyze_sentiment(text: str, client) -> dict:
    response = client.chat.completions.create(...)
    raw = response.choices[0].message.content
    
    # In development: log the raw output
    logger.debug(f"Raw LLM output: {raw!r}")
    
    return parse_and_process(raw)

# After capturing real outputs, use them as test cases:

# tests/data/real_llm_outputs.json
[
    {
        "id": "real_001",
        "input": "This product is fantastic",
        "raw_output": "{\n    \"sentiment\": \"positive\",\n    \"score\": 0.95,\n    \"explanation\": \"The text clearly uses positive adjectives.\",\n    \"keywords\": [\n        \"fantastic\"\n    ]\n}",
        "expected_sentiment": "positive"
    },
    {
        "id": "real_002",
        "input": "The service was horrible",
        "raw_output": "```json\n{\"sentiment\": \"negative\", \"score\": 0.05, \"explanation\": \"Very negative adjective.\", \"keywords\": [\"horrible\"]}\n```",
        "expected_sentiment": "negative"
    }
]

# Test using the captured real outputs:
import json
from pathlib import Path

def load_real_outputs():
    path = Path("tests/data/real_llm_outputs.json")
    with open(path) as f:
        return json.load(f)

@pytest.mark.parametrize("case", load_real_outputs(), ids=lambda c: c["id"])
def test_parser_handles_real_outputs(case):
    """The parser correctly handles the real outputs captured from the LLM."""
    result = process_llm_response(case["raw_output"])
    assert result["sentiment"] == case["expected_sentiment"]

Parser coverage: reaching 100%

Parsers are excellent candidates for 100% coverage:

# Run with coverage for the parser:
pytest tests/test_parsers.py --cov=app.parsers --cov-report=term-missing -v

# Expected output:
# Name              Stmts   Miss  Cover   Missing
# -----------------------------------------------
# app/parsers.py       45      2    96%   38-39

# If there are uncovered lines, add the test that exercises them:
# Line 38-39: the "nested JSON with subobjects" branch

def test_parse_nested_json():
    """Covers the branch for JSON with nested objects."""
    raw = '{"sentiment": "positive", "metadata": {"source": "twitter", "length": 50}}'
    result = parse_sentiment_response(raw)
    assert result["metadata"]["source"] == "twitter"

Anti-patterns in parser testing

# ❌ Anti-pattern 1: Test only the happy path
def test_parser_bad():
    result = parse_sentiment_response('{"sentiment": "positive"}')
    assert result["sentiment"] == "positive"
    # Only tests the simplest format — not the markdown, not the errors

# ✅ Correct: test multiple formats
@pytest.mark.parametrize("raw,expected", VALID_PARSER_CASES)
def test_parser_good(raw, expected):
    assert parse_sentiment_response(raw) == expected

# ❌ Anti-pattern 2: Not testing errors
def test_processor_bad():
    result = process_sentiment_output({"sentiment": "positive", "score": 0.9})
    assert result["sentiment"] == "positive"
    # Doesn't test out-of-range score, invalid sentiment, missing fields

# ✅ Correct: test all edge cases
def test_processor_good():
    assert process_sentiment_output({"sentiment": "invalid"})["sentiment"] == "neutral"
    assert process_sentiment_output({"score": 1.5})["score"] == 1.0
    assert process_sentiment_output({})["keywords"] == []

# ❌ Anti-pattern 3: Test parser + LLM together
def test_full_integration_bad():
    # Calls the real LLM to test the parser
    real_client = openai.OpenAI()
    result = analyze_sentiment("text", client=real_client)
    assert "sentiment" in result
    # Mixes responsibilities: the test can fail because of the LLM, not the parser

# ✅ Correct: test the parser with direct inputs
def test_parser_isolated():
    raw_from_llm = '{"sentiment": "positive", "score": 0.9}'
    result = parse_sentiment_response(raw_from_llm)
    assert result["sentiment"] == "positive"
    # Tests only the parser — no dependency on the LLM

Comparison: Parser vs complete chain

Conclusion: Testing the parser and processor in isolation is faster, cheaper, and covers more cases than testing the complete chain.

Exercises

Exercise 1: Entity parser with regex

The LLM returns text in this format: "Entities: Juan García (PERSON), January 15 (DATE), Madrid (LOCATION)". Write the parser and 5 tests.

import re
from typing import Optional

def parse_entities_response(raw: str) -> list[dict]:
    """
    Parses LLM responses that list entities.
    
    Expected format: "Name (TYPE), Name2 (TYPE2)"
    Returns: [{"entity": "Name", "type": "TYPE"}, ...]
    """
    if not raw or not raw.strip():
        return []
    
    # Remove prefixes like "Entities: ", "The entities are: ", etc.
    text = re.sub(r'^[^:]+:\s*', '', raw.strip())
    
    # Pattern: text (TYPE)
    pattern = r'([^,(]+?)\s*\(([^)]+)\)'
    matches = re.findall(pattern, text)
    
    return [
        {"entity": entity.strip(), "type": entity_type.strip().upper()}
        for entity, entity_type in matches
        if entity.strip() and entity_type.strip()
    ]

# Tests:
@pytest.mark.parametrize("raw,expected", [
    (
        "Juan García (PERSON), January 15 (DATE)",
        [{"entity": "Juan García", "type": "PERSON"}, {"entity": "January 15", "type": "DATE"}]
    ),
    (
        "Entities: Madrid (LOCATION)",
        [{"entity": "Madrid", "type": "LOCATION"}]
    ),
    (
        "",
        []
    ),
    (
        "Text with no entities",
        []
    ),
    (
        "person (type1), other entity (type2), third (type3)",
        [
            {"entity": "person", "type": "TYPE1"},
            {"entity": "other entity", "type": "TYPE2"},
            {"entity": "third", "type": "TYPE3"}
        ]
    ),
])
def test_parse_entities(raw, expected):
    assert parse_entities_response(raw) == expected

Exercise 2: Processor with graceful truncation

Write the output processor for support ticket analysis. It must handle:

• priority must be one of ["high", "medium", "low"], default "medium"
• title maximum 100 characters, truncated with "..." if longer
• tags list of strings, maximum 5 elements, empty by default

def process_ticket_analysis(raw: dict) -> dict:
    VALID_PRIORITIES = {"high", "medium", "low"}
    
    # Priority with fallback
    priority = str(raw.get("priority", "")).strip().lower()
    priority = priority if priority in VALID_PRIORITIES else "medium"
    
    # Title with graceful truncation
    title = str(raw.get("title", "")).strip()
    if len(title) > 100:
        title = title[:97] + "..."
    
    # Tags: list of strings, maximum 5
    raw_tags = raw.get("tags", [])
    if isinstance(raw_tags, list):
        tags = [str(t).strip() for t in raw_tags if t and str(t).strip()][:5]
    else:
        tags = []
    
    return {"priority": priority, "title": title, "tags": tags}

# Tests:
def test_process_ticket_valid():
    result = process_ticket_analysis({"priority": "high", "title": "Server down", "tags": ["prod", "critical"]})
    assert result == {"priority": "high", "title": "Server down", "tags": ["prod", "critical"]}

def test_process_ticket_invalid_priority():
    assert process_ticket_analysis({"priority": "urgent"})["priority"] == "medium"

def test_process_ticket_long_title():
    long_title = "A" * 150
    result = process_ticket_analysis({"title": long_title})
    assert len(result["title"]) == 100
    assert result["title"].endswith("...")

def test_process_ticket_too_many_tags():
    result = process_ticket_analysis({"tags": ["t1", "t2", "t3", "t4", "t5", "t6"]})
    assert len(result["tags"]) == 5

def test_process_ticket_empty():
    result = process_ticket_analysis({})
    assert result == {"priority": "medium", "title": "", "tags": []}

Exercise 3: Test with real captured outputs

Imagine you captured these 3 real LLM outputs. Write the parametrized tests:

REAL_OUTPUTS = [
    {
        "raw": "{\n  \"sentiment\": \"positive\",\n  \"score\": 0.92,\n  \"explanation\": \"Uses positive adjectives.\"\n}",
        "expected_sentiment": "positive",
        "expected_score_range": (0.8, 1.0)
    },
    {
        "raw": "```json\n{\"sentiment\": \"NEGATIVE\", \"score\": 0.08}\n```",
        "expected_sentiment": "negative",
        "expected_score_range": (0.0, 0.2)
    },
    {
        "raw": "The analysis: {\"sentiment\": \"neutral\", \"score\": 0.5, \"extra\": \"ignored\"}",
        "expected_sentiment": "neutral",
        "expected_score_range": (0.4, 0.6)
    }
]

@pytest.mark.parametrize("case", REAL_OUTPUTS, ids=[f"real_{i}" for i in range(len(REAL_OUTPUTS))])
def test_parser_with_real_captured_outputs(case):
    """
    Parametrized tests with real LLM outputs.
    These cases were captured during development to ensure
    that the parser handles the real formats correctly.
    """
    result = process_llm_response(case["raw"])
    
    assert result["sentiment"] == case["expected_sentiment"]
    
    min_score, max_score = case["expected_score_range"]
    assert min_score <= result["score"] <= max_score, \
        f"Score {result['score']} out of range [{min_score}, {max_score}]"

Exercise 4: Coverage

Run coverage on the parser and explain how you'd reach 100%:

pytest tests/test_parsers.py --cov=app.parsers --cov-report=term-missing -v

Exercise 5: Test the complete pipeline

Write a test that exercises the complete parse + process pipeline for a ticket classification output:

The raw output can come in any of these formats:

1. '{"priority": "HIGH", "title": "Server down", "tags": ["prod"]}'
2. '```json\n{"priority": "medium", "title": "Login issue"}\n```'

@pytest.mark.parametrize("raw,expected_priority,expected_title_contains", [
    (
        '{"priority": "HIGH", "title": "Server down", "tags": ["prod"]}',
        "high",
        "Server down"
    ),
    (
        '```json\n{"priority": "medium", "title": "Login issue"}\n```',
        "medium",
        "Login issue"
    ),
])
def test_ticket_pipeline_complete(raw, expected_priority, expected_title_contains):
    """Complete pipeline: parse ticket JSON + process."""
    # Step 1: Parser
    parsed = parse_ticket_response(raw)
    assert isinstance(parsed, dict)
    
    # Step 2: Processor
    result = process_ticket_analysis(parsed)
    
    # Assert: complete pipeline
    assert result["priority"] == expected_priority
    assert expected_title_contains in result["title"]
    assert isinstance(result["tags"], list)

Summary

• Parsers and processors are 100% deterministic — test them without mocks, with no API cost
• parametrize for multiple formats — the LLM produces real format variations
• Critical edge cases: empty, malformed, truncated, with markdown, with preceding text
• Output processors: normalization, range clamping, defaults for missing fields
• 100% coverage is achievable for parsers — it's the ideal component for this
• Capture real LLM outputs and use them as test cases for realistic coverage

Additional resources

1. pytest.mark.parametrize — For multiple input cases
2. pytest.raises — For testing expected exceptions
3. Python json module — Reference for the standard JSON module
4. Python re module — Regex for text parsing
5. Pydantic v2 — Validators — For structured validation
6. pytest-cov — To measure parser coverage
7. Hypothesis — Property-based testing for parsers (covered in Module 3)