การประเมินโมเดลและการตรวจสอบไขว้

การแบ่งชุดฝึก/ตรวจสอบ/ทดสอบ

from sklearn.model_selection import train_test_split
import numpy as np

X = np.random.rand(1000, 10)
y = (X[:,0] > 0.5).astype(int)

X_tr, X_rest, y_tr, y_rest = train_test_split(X, y, test_size=0.3)
X_val, X_te, y_val, y_te = train_test_split(X_rest, y_rest, test_size=0.5)

คะแนนความถูกต้อง

from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

X, y = make_classification(random_state=0)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=0)
model = LogisticRegression().fit(X_tr, y_tr)
print("Accuracy:", accuracy_score(y_te, model.predict(X_te)))

ความแม่นยำ ความครอบคลุม และ F1

from sklearn.metrics import precision_score, recall_score, f1_score, classification_report

# Use classification_report for a full summary:
print(classification_report(y_te, model.predict(X_te)))
# shows precision, recall, f1 for each class

เมทริกซ์ความสับสน

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

cm = confusion_matrix(y_te, model.predict(X_te))
print(cm)
# [[TN FP]
#  [FN TP]]

เส้นโค้ง ROC และ AUC

from sklearn.metrics import roc_auc_score, roc_curve

y_prob = model.predict_proba(X_te)[:,1]
print("AUC:", roc_auc_score(y_te, y_prob))

fpr, tpr, thresholds = roc_curve(y_te, y_prob)
# plot fpr vs tpr for the full ROC curve

ตัวชี้วัดการถดถอย

from sklearn.metrics import mean_squared_error, r2_score

y_pred = regression_model.predict(X_te)
mse = mean_squared_error(y_te, y_pred)
print("RMSE:", mse**0.5)
print("R²:", r2_score(y_te, y_pred))

การตรวจสอบไขว้แบบ k โฟลด์

from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification

X, y = make_classification(random_state=0)
scores = cross_val_score(LogisticRegression(), X, y, cv=10, scoring="accuracy")
print(f"{scores.mean():.3f} ± {scores.std():.3f}")

StratifiedKFold

from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification

X, y = make_classification(weights=[0.9,0.1], random_state=0)
cv = StratifiedKFold(n_splits=5)
scores = cross_val_score(LogisticRegression(), X, y, cv=cv, scoring="f1")
print("Stratified F1:", scores.mean())

ความสัมพันธ์ระหว่างอคติกับความแปรปรวน

# Underfitting: low train score AND low val score
# → increase model complexity, add features

# Overfitting: high train score, low val score
# → regularise, reduce complexity, get more data

from sklearn.model_selection import validation_curve
import numpy as np

train_sc, val_sc = validation_curve(DecisionTreeClassifier(), X, y, param_name="max_depth", param_range=range(1,10))

การตรวจสอบไขว้แบบซ้อน

from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.svm import SVC
from sklearn.datasets import make_classification

X, y = make_classification(random_state=0)
gs = GridSearchCV(SVC(), {"C":[0.1,1,10]}, cv=5)
outer_scores = cross_val_score(gs, X, y, cv=5)
print("Nested CV accuracy:", outer_scores.mean())

การปรับเทียบ

from sklearn.calibration import CalibratedClassifierCV
from sklearn.svm import SVC
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

X, y = make_classification(random_state=0)
X_tr, X_te, y_tr, y_te = train_test_split(X, y)

cal = CalibratedClassifierCV(SVC(), cv=5).fit(X_tr, y_tr)
print(cal.predict_proba(X_te[:3]))

ตรวจสอบความเข้าใจ

สรุปทบทวน