Outlier Detection

Detecting anomalies

def detect_outliers(embeddings, threshold=2.0):
    """Detect outliers using distance to the centroid"""
    centroid = np.mean(embeddings, axis=0)
    
    # Distances to the centroid
    distances = [np.linalg.norm(emb - centroid) for emb in embeddings]
    
    # Outliers = distance > mean + threshold*std
    mean_dist = np.mean(distances)
    std_dist = np.std(distances)
    outlier_threshold = mean_dist + threshold * std_dist
    
    outliers = []
    for i, dist in enumerate(distances):
        if dist > outlier_threshold:
            outliers.append(i)
    
    return outliers

# Example
outlier_indices = detect_outliers(doc_embeddings, threshold=2.0)
print(f"Outliers: {len(outlier_indices)} documents")

---

Isolation Forest

from sklearn.ensemble import IsolationForest

# More robust than a simple distance
clf = IsolationForest(contamination=0.1, random_state=42)
predictions = clf.fit_predict(embeddings)

# -1 = outlier, 1 = normal
outliers = np.where(predictions == -1)[0]
print(f"Outliers: {len(outliers)}")

---

Applications

Data quality:

# Detect misclassified documents
category_embs = [get_emb(doc) for doc in category_docs]
outliers = detect_outliers(category_embs)

# Manually review the outliers (possible mislabel)
for idx in outliers:
    print(f"Potential mislabel: {category_docs[idx]}")

Spam detection:

# Legitimate emails have similar embeddings
# Spam = outlier
legitimate_embs = [...]
new_email_emb = get_emb(new_email)

# Is it an outlier?
is_spam = is_outlier(new_email_emb, legitimate_embs)

---

Summary

• ✅ Outlier: An embedding very different from the rest
• ✅ Detection: Distance to centroid > threshold
• ✅ Isolation Forest: More robust
• ✅ Use case: Data quality, spam detection

---

Module 6 - Capsule 07