การเลือก k: วิธีข้อศอกและเส้นโค้งการตรวจสอบ

# k=1: memorises training set perfectly
# Training accuracy = 100%, test accuracy low (overfitting)

# k=N (all neighbors): always predicts majority class
# Training accuracy = majority fraction (underfitting)

# Optimal k: somewhere in between
# Maximises test/validation accuracy

from sklearn.neighbors import KNeighborsClassifier
print('k=1  overfits (memorises noise)')
print('k=N  underfits (ignores all variation)')
print('Best k: maximises cross-validated accuracy')

from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_iris

X, y = load_iris(return_X_y=True)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_tr = scaler.fit_transform(X_train)
X_v  = scaler.transform(X_val)

val_scores = []
for k in range(1, 31):
    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(X_tr, y_train)
    val_scores.append(knn.score(X_v, y_val))

best_k = val_scores.index(max(val_scores)) + 1
print('Best k:', best_k, 'with accuracy:', max(val_scores).round(3))

import matplotlib.pyplot as plt
import numpy as np

k_values = range(1, 31)
# val_scores computed from previous sweep
plt.figure(figsize=(10, 5))
plt.plot(k_values, val_scores, marker='o', label='Validation accuracy')
plt.axvline(x=best_k, color='r', linestyle='--', label=f'Best k={best_k}')
plt.xlabel('k (Number of Neighbors)')
plt.ylabel('Validation Accuracy')
plt.title('KNN Validation Curve')
plt.legend()
plt.grid(True)
plt.show()

from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
import numpy as np

X, y = load_iris(return_X_y=True)

cv_means, cv_stds = [], []
for k in range(1, 31):
    pipe = Pipeline([('sc', StandardScaler()),
                     ('knn', KNeighborsClassifier(n_neighbors=k))])
    scores = cross_val_score(pipe, X, y, cv=10)
    cv_means.append(scores.mean())
    cv_stds.append(scores.std())

best_k_cv = np.argmax(cv_means) + 1
print('Best k by CV:', best_k_cv, 'mean accuracy:', max(cv_means).round(3))

import matplotlib.pyplot as plt
import numpy as np

k_range = range(1, 31)
error_rates = [1 - acc for acc in cv_means]  # Convert accuracy to error

plt.figure(figsize=(10, 5))
plt.plot(list(k_range), error_rates, marker='o')
plt.xlabel('k')
plt.ylabel('Cross-Validated Error Rate')
plt.title('Elbow Method for Optimal k')
plt.grid(True)

# Mark the elbow visually
plt.axvline(x=best_k_cv, color='r', linestyle='--', label=f'Elbow at k={best_k_cv}')
plt.legend()
plt.show()

from sklearn.model_selection import validation_curve
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
import numpy as np

pipe = Pipeline([('sc', StandardScaler()), ('knn', KNeighborsClassifier())])

train_scores, val_scores = validation_curve(
    pipe, X, y,
    param_name='knn__n_neighbors',
    param_range=range(1, 31),
    cv=10, scoring='accuracy'
)

train_mean = np.mean(train_scores, axis=1)
val_mean   = np.mean(val_scores, axis=1)

print('Best k:', np.argmax(val_mean) + 1)

import matplotlib.pyplot as plt
import numpy as np

fig, ax = plt.subplots(figsize=(10, 5))
k_range = list(range(1, 31))

ax.plot(k_range, train_mean, label='Training accuracy', color='blue')
ax.plot(k_range, val_mean, label='Validation accuracy', color='orange')
ax.fill_between(k_range,
    np.mean(train_scores, axis=1) - np.std(train_scores, axis=1),
    np.mean(train_scores, axis=1) + np.std(train_scores, axis=1),
    alpha=0.1, color='blue')

ax.set_xlabel('k')
ax.set_ylabel('Accuracy')
ax.set_title('Bias-Variance Trade-off: KNN Validation Curve')
ax.legend()
plt.show()

# Best practice for binary classification: pick odd k
# For multi-class (C classes): avoid multiples of C

def recommend_k(k_optimal, n_classes):
    if n_classes == 2:
        # Make odd
        return k_optimal if k_optimal % 2 == 1 else k_optimal + 1
    else:
        # Avoid multiples of n_classes
        while k_optimal % n_classes == 0:
            k_optimal += 1
        return k_optimal

print('Binary, k=4 -> recommended:', recommend_k(4, 2))  # 5
print('3-class, k=6 -> recommended:', recommend_k(6, 3)) # 7

from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier

pipe = Pipeline([
    ('sc', StandardScaler()),
    ('knn', KNeighborsClassifier())
])

param_grid = {
    'knn__n_neighbors': list(range(1, 31, 2)),  # odd values 1-29
    'knn__weights': ['uniform', 'distance']
}

grid = GridSearchCV(pipe, param_grid, cv=10, scoring='accuracy', n_jobs=-1)
grid.fit(X_train, y_train)

print('Best k:', grid.best_params_['knn__n_neighbors'])
print('Best weights:', grid.best_params_['knn__weights'])
print('Best CV accuracy:', grid.best_score_.round(3))

import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# Select best k from validation
best_k = 11  # determined from CV

# Retrain on full training data
final_model = Pipeline([
    ('sc', StandardScaler()),
    ('knn', KNeighborsClassifier(n_neighbors=best_k, weights='distance'))
])
final_model.fit(X_train, y_train)

# Evaluate once on held-out test set
test_accuracy = final_model.score(X_test, y_test)
print(f'Final test accuracy with k={best_k}: {test_accuracy:.3f}')

import numpy as np

N_train = 1000  # training samples

# Rule of thumb starting point
k_start = int(np.sqrt(N_train))
print(f'sqrt(N) starting point: k = {k_start}')

# Then sweep around this value
k_candidates = list(range(max(1, k_start - 10), k_start + 11, 2))
print('Candidates to sweep:', k_candidates)

# AVOID:
# knn.score(X_train, y_train) -- always pick k=1
# Not including scaler in pipeline before CV