KNN 작동 방식: 거리, 이웃, 투표

The Core Intuition Behind KNN

# Conceptual pseudocode
def knn_predict(X_train, y_train, x_new, k=3):
    # 1. Compute distance from x_new to every training point
    distances = [euclidean(x_new, x_i) for x_i in X_train]
    # 2. Find indices of k smallest distances
    nearest = sorted(range(len(distances)), key=lambda i: distances[i])[:k]
    # 3. Majority vote among k neighbors
    votes = [y_train[i] for i in nearest]
    return max(set(votes), key=votes.count)

Euclidean Distance: The Default Metric

import numpy as np

def euclidean_distance(a, b):
    return np.sqrt(np.sum((a - b) ** 2))

point_a = np.array([1.0, 2.0])
point_b = np.array([4.0, 6.0])

dist = euclidean_distance(point_a, point_b)
print('Euclidean distance:', dist)  # 5.0

# Verify with numpy
print('Using numpy:', np.linalg.norm(point_a - point_b))

Finding the k Nearest Neighbors

import numpy as np

# Training data
X_train = np.array([[1,2],[2,3],[3,1],[6,5],[7,7],[8,6]])
y_train = np.array([0, 0, 0, 1, 1, 1])  # 0=class A, 1=class B

# Query point
x_new = np.array([4, 4])

# Distances to all training points
dists = np.linalg.norm(X_train - x_new, axis=1)
print('Distances:', dists.round(2))

# Indices of 3 nearest
k = 3
nearest_idx = np.argsort(dists)[:k]
print('3 nearest labels:', y_train[nearest_idx])

Classification by Majority Vote

from collections import Counter
import numpy as np

neighbor_labels = np.array([0, 0, 1])  # 2 votes for class 0, 1 for class 1

# Majority vote
vote_counts = Counter(neighbor_labels)
prediction = vote_counts.most_common(1)[0][0]
print('Predicted class:', prediction)  # 0

# For regression: average instead of vote
neighbor_values = np.array([15.2, 18.7, 14.1])
prediction_reg = np.mean(neighbor_values)
print('Regression prediction:', prediction_reg.round(2))

KNN with scikit-learn: KNeighborsClassifier

from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

knn = KNeighborsClassifier(n_neighbors=5, weights='uniform')
knn.fit(X_train_s, y_train)
print('Test accuracy:', knn.score(X_test_s, y_test).round(3))

Visualising the Decision Boundary

import numpy as np
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier

# Create 2D mesh for decision boundary
def plot_decision_boundary(clf, X, y):
    h = 0.02
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(
        np.arange(x_min, x_max, h),
        np.arange(y_min, y_max, h)
    )
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, alpha=0.4)
    plt.scatter(X[:, 0], X[:, 1], c=y)

Why Feature Scaling Is Critical for KNN

from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_breast_cancer

X, y = load_breast_cancer(return_X_y=True)

# Without scaling
knn_raw = KNeighborsClassifier(n_neighbors=5)
raw_score = cross_val_score(knn_raw, X, y, cv=5).mean()

# With scaling inside pipeline
pipe = Pipeline([('sc', StandardScaler()), ('knn', KNeighborsClassifier(n_neighbors=5))])
scaled_score = cross_val_score(pipe, X, y, cv=5).mean()

print(f'Without scaling: {raw_score:.3f}')
print(f'With scaling:    {scaled_score:.3f}')

Getting Neighbor Distances and Indices

from sklearn.neighbors import KNeighborsClassifier
import numpy as np

X_train = np.array([[1,2],[2,3],[5,5],[8,7]])
y_train = np.array([0, 0, 1, 1])

knn = KNeighborsClassifier(n_neighbors=2)
knn.fit(X_train, y_train)

x_query = np.array([[4, 4]])
distances, indices = knn.kneighbors(x_query)

print('Neighbor indices:', indices)
print('Distances to neighbors:', distances.round(2))
print('Neighbor labels:', y_train[indices[0]])

Weighted Voting by Distance

from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_digits
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

X, y = load_digits(return_X_y=True)

uniform_pipe = Pipeline([('sc', StandardScaler()),
                         ('knn', KNeighborsClassifier(n_neighbors=7, weights='uniform'))])
distance_pipe = Pipeline([('sc', StandardScaler()),
                          ('knn', KNeighborsClassifier(n_neighbors=7, weights='distance'))])

print('Uniform:  ', cross_val_score(uniform_pipe, X, y, cv=5).mean().round(3))
print('Distance: ', cross_val_score(distance_pipe, X, y, cv=5).mean().round(3))

Predicting Class Probabilities

from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
import numpy as np

X_train = np.array([[1,1],[1,2],[5,5],[6,5],[5,6]])
y_train = np.array([0, 0, 1, 1, 1])

scaler = StandardScaler()
X_s = scaler.fit_transform(X_train)

knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(X_s, y_train)

x_new = scaler.transform([[3, 3]])
proba = knn.predict_proba(x_new)
print('Class probabilities:', proba)
print('Predicted class:', knn.predict(x_new))

KNN Strengths and Weaknesses

# Summary of KNN trade-offs
strengths = [
    'No training time -- all computation at prediction',
    'No assumptions about data distribution',
    'Naturally handles multi-class classification',
    'Non-linear decision boundary',
]

weaknesses = [
    'Slow prediction: O(N*d) per query',
    'High memory: stores all training data',
    'Sensitive to irrelevant and scaled features',
    'Poor in very high dimensions (curse of dimensionality)',
]

for s in strengths: print('+', s)
for w in weaknesses: print('-', w)

Quick Check

Lesson Recap