การเลือกตัวชี้วัดที่เหมาะสมสำหรับปัญหาทางธุรกิจ

# The metric choice depends on error costs

# Scenario 1: Email spam filter
# FP (ham marked as spam) -> user misses important email: HIGH cost
# FN (spam reaches inbox) -> user is annoyed: LOW cost
# -> Optimise PRECISION (avoid false alarms)

# Scenario 2: Cancer screening
# FP (healthy patient flagged) -> unnecessary follow-up: LOW cost
# FN (cancer patient missed) -> delayed treatment: HIGH cost
# -> Optimise RECALL (catch all true cases)

# Scenario 3: House price prediction
# No asymmetric cost -> use RMSE or R^2
print('Define your error cost structure BEFORE choosing a metric')

import numpy as np
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    average_precision_score, roc_auc_score
)

# Simulated fraud: 1000 transactions, 10 are fraud (1%)
y_true = np.array([0]*990 + [1]*10)

# Model A: always predict legitimate
y_dummy = np.zeros(1000, dtype=int)

print('Dummy model (always legitimate):')
print(f'  Accuracy: {accuracy_score(y_true, y_dummy):.3f}')  # 99% -- misleading!
print(f'  Recall:   {recall_score(y_true, y_dummy):.3f}')    # 0.0 -- catches no fraud!
print()
print('For fraud detection: use Recall, F1, or AUC-PR')

from sklearn.metrics import recall_score, precision_score, f1_score
import numpy as np

# Simulated diagnosis: 1000 patients, 50 have the disease
y_true = np.array([0]*950 + [1]*50)

# High-recall model: flags 200 positives, 45 are true positive
y_high_recall = np.array([0]*900 + [1]*100)  # approximate
y_pred_labels = np.zeros(1000, dtype=int)
y_pred_labels[-100:] = 1  # flag last 100 (including 50 true)

TP = 50  # catches all 50 true positives
FP = 50  # but also flags 50 healthy patients
FN = 0   # misses no true positives

recall = TP / (TP + FN)
prec   = TP / (TP + FP)
print(f'Medical screening model: Recall={recall:.2f}, Precision={prec:.2f}')
print('Recall 1.0 is ideal: no cancer patient missed.')

import numpy as np

def precision_at_k(recommended, relevant, k):
    '''Of top k recommendations, fraction that are relevant'''
    top_k = recommended[:k]
    relevant_set = set(relevant)
    hits = sum(1 for item in top_k if item in relevant_set)
    return hits / k

# Example: recommend 10 items, 3 are truly relevant
recommended = ['item_A', 'item_B', 'item_C', 'item_D', 'item_E',
               'item_F', 'item_G', 'item_H', 'item_I', 'item_J']
relevant     = ['item_A', 'item_C', 'item_G']

for k in [3, 5, 10]:
    print(f'Precision@{k}: {precision_at_k(recommended, relevant, k):.2f}')

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
import numpy as np

# Simulated house prices (in $1000s)
y_true = np.array([250, 380, 510, 420, 650, 290, 730, 480])
y_pred = np.array([265, 360, 525, 435, 620, 310, 700, 465])

mae  = mean_absolute_error(y_true, y_pred)
rmse = np.sqrt(mean_squared_error(y_true, y_pred))
r2   = r2_score(y_true, y_pred)

print('House Price Model Performance:')
print(f'  MAE:  ${mae:.1f}k -- typical error for a buyer seeing estimates')
print(f'  RMSE: ${rmse:.1f}k -- influenced by occasional large errors')
print(f'  R^2:  {r2:.3f} -- fraction of price variance explained')

# Translating ML metric improvements to business value

# Fraud detection example:
new_cases_per_month = 10000   # Fraud transactions per month
avg_fraud_value = 500         # Average fraud value ($)
current_recall = 0.70         # Model catches 70% of fraud
proposed_recall = 0.80        # Improved model catches 80%

current_caught  = new_cases_per_month * current_recall  * avg_fraud_value
proposed_caught = new_cases_per_month * proposed_recall * avg_fraud_value

business_value = proposed_caught - current_caught
print(f'Recall improvement: {current_recall:.0%} -> {proposed_recall:.0%}')
print(f'Business value: ${business_value:,.0f}/month additional fraud prevented')

from sklearn.metrics import f1_score
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=42)

clf = LogisticRegression(max_iter=10000)
clf.fit(X_tr, y_tr)
proba = clf.predict_proba(X_te)[:, 1]

# Find threshold that maximises F1
best_f1, best_thresh = 0, 0.5
for thresh in np.linspace(0.1, 0.9, 81):
    pred = (proba >= thresh).astype(int)
    f1 = f1_score(y_te, pred, zero_division=0)
    if f1 > best_f1:
        best_f1, best_thresh = f1, thresh

print(f'Default (0.5) F1:  {f1_score(y_te, (proba>=0.5).astype(int)):.4f}')
print(f'Optimised F1:      {best_f1:.4f} at threshold {best_thresh:.2f}')

from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
)
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

X, y = load_breast_cancer(return_X_y=True)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=42)
X_tr_s = StandardScaler().fit_transform(X_tr)
X_te_s = StandardScaler().fit(X_tr).transform(X_te)

for name, clf in [('Dummy', DummyClassifier(strategy='most_frequent')),
                   ('LogReg', LogisticRegression(max_iter=1000))]:
    clf.fit(X_tr_s, y_tr)
    pred = clf.predict(X_te_s)
    prob = clf.predict_proba(X_te_s)[:, 1]
    print(f'\n{name}:')
    print(f'  Accuracy={accuracy_score(y_te,pred):.3f}, '
          f'Precision={precision_score(y_te,pred):.3f}, '
          f'Recall={recall_score(y_te,pred):.3f}, '
          f'F1={f1_score(y_te,pred):.3f}, '
          f'AUC={roc_auc_score(y_te,prob):.3f}')

from sklearn.calibration import CalibrationDisplay
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

X, y = load_breast_cancer(return_X_y=True)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, random_state=42)

fig, ax = plt.subplots(figsize=(8, 6))
for name, clf in [('LogReg', LogisticRegression(max_iter=1000)),
                   ('RandomForest', RandomForestClassifier(n_estimators=100))]:
    clf.fit(X_tr, y_tr)
    CalibrationDisplay.from_estimator(clf, X_te, y_te, ax=ax, name=name, n_bins=10)

ax.set_title('Calibration Curves')
plt.show()

def select_metric(task, balanced, fp_cost_high, fn_cost_high, need_proba_cal):
    if task == 'regression':
        return 'MAE + RMSE + R^2 (report all three)'
    # Classification:
    if not balanced:
        if fn_cost_high and fp_cost_high:
            return 'F1 + AUC-PR (balance precision and recall)'
        elif fn_cost_high:
            return 'Recall + AUC-PR (minimise missed positives)'
        elif fp_cost_high:
            return 'Precision + AUC-ROC (minimise false alarms)'
    else:
        return 'Accuracy + F1 + AUC-ROC'
    return 'AUC-ROC + F1'

print(select_metric('classification', False, False, True, False))  # Recall
print(select_metric('classification', True, True, True, False))    # Accuracy+F1
print(select_metric('regression', True, False, False, False))      # MAE+RMSE+R^2

# Minimal model card template
model_card = {
    'model_name': 'Fraud Detector v2.1',
    'task': 'Binary classification (fraud / legitimate)',
    'primary_metric': 'Recall at FPR=0.03',
    'metric_rationale': 'Missing fraud (FN) costs $500 avg; false alarm (FP) costs $5 investigation',
    'baseline': {'model': 'DummyClassifier(most_frequent)', 'recall': 0.0, 'fpr': 0.0},
    'deployed_model': {'algorithm': 'XGBoost', 'recall': 0.82, 'fpr': 0.03},
    'business_kpi': '$2.3M/month fraud prevented vs rule-based system',
    'evaluation_date': '2026-06-20',
    'training_data': 'Transactions 2024-01 to 2025-12, 5M samples'
}

for key, value in model_card.items():
    print(f'{key}: {value}')