Outlier Detection

Detectar anomalías

def detect_outliers(embeddings, threshold=2.0):
    """Detectar outliers usando distancia al centroid"""
    centroid = np.mean(embeddings, axis=0)
    
    # Distancias al centroid
    distances = [np.linalg.norm(emb - centroid) for emb in embeddings]
    
    # Outliers = distancia > mean + threshold*std
    mean_dist = np.mean(distances)
    std_dist = np.std(distances)
    outlier_threshold = mean_dist + threshold * std_dist
    
    outliers = []
    for i, dist in enumerate(distances):
        if dist > outlier_threshold:
            outliers.append(i)
    
    return outliers

# Ejemplo
outlier_indices = detect_outliers(doc_embeddings, threshold=2.0)
print(f"Outliers: {len(outlier_indices)} documentos")

---

Isolation Forest

from sklearn.ensemble import IsolationForest

# Más robusto que distancia simple
clf = IsolationForest(contamination=0.1, random_state=42)
predictions = clf.fit_predict(embeddings)

# -1 = outlier, 1 = normal
outliers = np.where(predictions == -1)[0]
print(f"Outliers: {len(outliers)}")

---

Applications

Data quality:

# Detectar documentos mal clasificados
category_embs = [get_emb(doc) for doc in category_docs]
outliers = detect_outliers(category_embs)

# Revisar manualmente outliers (posible mislabel)
for idx in outliers:
    print(f"Potential mislabel: {category_docs[idx]}")

Spam detection:

# Emails legítimos tienen embeddings similares
# Spam = outlier
legitimate_embs = [...]
new_email_emb = get_emb(new_email)

# ¿Es outlier?
is_spam = is_outlier(new_email_emb, legitimate_embs)

---

Resumen

• ✅ Outlier: Embedding muy diferente del resto
• ✅ Detection: Distancia al centroid > threshold
• ✅ Isolation Forest: Más robusto
• ✅ Use case: Data quality, spam detection

---

Módulo 6 - Cápsula 07