Fault Tolerance

Health Checks

def health_check():
    """Check system health"""
    checks = {
        'openai_api': check_openai(),
        'redis': check_redis(),
        'local_model': check_local_model()
    }
    
    return {
        'status': 'healthy' if all(checks.values()) else 'degraded',
        'checks': checks
    }

def check_openai():
    try:
        client.embeddings.create(
            model="text-embedding-3-small",
            input="test"
        )
        return True
    except:
        return False

---

Graceful Shutdown

import signal
import sys

class EmbeddingService:
    def __init__(self):
        self.running = True
        signal.signal(signal.SIGTERM, self.shutdown)
    
    def shutdown(self, signum, frame):
        """Graceful shutdown"""
        print("Shutting down gracefully...")
        self.running = False
        
        # Finish processing the current batch
        self.finish_current_work()
        
        # Close connections
        self.redis.close()
        
        sys.exit(0)

---

Multi-Provider Failover

def get_embedding_with_failover(text):
    """Multi-provider with automatic failover"""
    providers = [
        ('openai', get_embedding_openai),
        ('cohere', get_embedding_cohere),
        ('local', get_embedding_sbert)
    ]
    
    for name, func in providers:
        try:
            return func(text)
        except Exception as e:
            logger.warning(f"{name} failed: {e}")
            continue
    
    raise Exception("All providers failed")

---

Summary

• ✅ Health checks: Monitor dependencies
• ✅ Graceful shutdown: Finish work before exit
• ✅ Multi-provider: Automatic failover
• ✅ Production: Zero downtime

---

Module 7 - Capsule 07