임베딩 군집화 및 시각화

임베딩을 클러스터링하는 이유

K-평균 클러스터링 개요

from sklearn.cluster import KMeans
import numpy as np

# corpus_embeddings: (n_docs, 1536) — pre-computed
corpus_embeddings = np.random.randn(200, 1536)  # placeholder

kmeans = KMeans(n_clusters=5, random_state=42, n_init='auto')
kmeans.fit(corpus_embeddings)

labels = kmeans.labels_
print(f'Cluster assignments: {labels[:10]}')
print(f'Unique clusters: {set(labels)}')

적절한 클러스터 수 선택하기

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import numpy as np

corpus_embeddings = np.random.randn(200, 50)  # placeholder (50D for speed)

best_k, best_score = 2, -1
for k in range(2, 11):
    km = KMeans(n_clusters=k, random_state=42, n_init='auto')
    labels = km.fit_predict(corpus_embeddings)
    score = silhouette_score(corpus_embeddings, labels, sample_size=100)
    print(f'k={k}: silhouette={score:.3f}')
    if score > best_score:
        best_score, best_k = score, k

print(f'Best k: {best_k}')

LLM으로 클러스터에 레이블 지정하기

from openai import OpenAI

client = OpenAI()

def label_cluster(cluster_docs, n_examples=3):
    examples = '\n'.join(f'- {d}' for d in cluster_docs[:n_examples])
    prompt = f'Given these documents, provide a 3-word category label:\n{examples}\nLabel:'
    response = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': prompt}],
        max_tokens=20
    )
    return response.choices[0].message.content.strip()

# Example usage:
# cluster_0_docs = [documents[i] for i in range(len(documents)) if labels[i] == 0]
# print(label_cluster(cluster_0_docs))

UMAP으로 차원 축소하기

import umap
import numpy as np

corpus_embeddings = np.random.randn(200, 1536)  # placeholder

reducer = umap.UMAP(
    n_components=2,
    metric='cosine',   # important for text embeddings
    random_state=42
)

embeddings_2d = reducer.fit_transform(corpus_embeddings)
print(f'Reduced shape: {embeddings_2d.shape}')  # (200, 2)

Matplotlib으로 클러스터 시각화하기

import matplotlib.pyplot as plt
import numpy as np

# Assume: embeddings_2d (n, 2), labels (n,), documents list
# Placeholder data:
np.random.seed(42)
embeddings_2d = np.random.randn(50, 2)
labels = np.random.randint(0, 5, 50)

fig, ax = plt.subplots(figsize=(10, 7))
scatter = ax.scatter(
    embeddings_2d[:, 0],
    embeddings_2d[:, 1],
    c=labels,
    cmap='tab10',
    s=60,
    alpha=0.8
)
plt.colorbar(scatter, label='Cluster')
ax.set_title('Document Embedding Clusters (UMAP)')
plt.tight_layout()
plt.savefig('/tmp/clusters.png', dpi=150)
print('Saved cluster plot')

중요한 UMAP 매개변수

계층적 클러스터링 대안

from sklearn.cluster import AgglomerativeClustering
import numpy as np

corpus_embeddings = np.random.randn(100, 50)  # placeholder

cluster = AgglomerativeClustering(
    n_clusters=None,
    distance_threshold=1.5,
    metric='cosine',
    linkage='average'
)
labels = cluster.fit_predict(corpus_embeddings)

n_clusters = len(set(labels))
print(f'Found {n_clusters} natural clusters')

이상치와 거의 중복된 문서 찾기

import numpy as np

def find_near_duplicates(corpus_vecs, threshold=0.95):
    # corpus_vecs assumed L2-normalized
    sim_matrix = corpus_vecs @ corpus_vecs.T  # (n, n)
    duplicates = []
    n = len(corpus_vecs)
    for i in range(n):
        for j in range(i + 1, n):
            if sim_matrix[i, j] >= threshold:
                duplicates.append((i, j, float(sim_matrix[i, j])))
    return duplicates

# pairs = find_near_duplicates(corpus_embeddings)
# print(f'Found {len(pairs)} near-duplicate pairs')

실용적인 클러스터링 작업 흐름

클러스터링의 한계

빠른 확인

레슨 요약