Рабочий процесс машинного обучения: от данных к предсказанию

The End-to-End ML Pipeline

Stage 1: Define the Problem

Stage 2: Collect and Load Data

import pandas as pd

# Load data from a CSV file
df = pd.read_csv('housing.csv')

# First inspection
print(df.shape)       # (rows, columns)
print(df.dtypes)      # data types per column
print(df.head())      # first 5 rows
print(df.describe())  # summary statistics

Stage 3: Exploratory Data Analysis

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

df = pd.read_csv('housing.csv')

# Distribution of target variable
df['price'].hist(bins=50)
plt.title('House Price Distribution')
plt.show()

# Correlation heat map
sns.heatmap(df.corr(), annot=True, cmap='coolwarm')
plt.show()

# Check for missing values
print(df.isnull().sum())

Stage 4: Preprocess the Data

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import pandas as pd
import numpy as np

df = pd.read_csv('housing.csv')
X = df.drop('price', axis=1)
y = df['price']

# Split first, then fit scaler ONLY on train
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)  # fit + transform on train
X_test_scaled = scaler.transform(X_test)         # transform only on test

Stage 5: Train the Model

from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor

# Simple baseline first
linear_model = LinearRegression()
linear_model.fit(X_train_scaled, y_train)

# More complex model
rf_model = RandomForestRegressor(n_estimators=100, random_state=42)
rf_model.fit(X_train_scaled, y_train)

print('Linear model trained.')
print('Random Forest trained.')

Stage 6: Evaluate the Model

from sklearn.metrics import mean_absolute_error, r2_score
import numpy as np

y_pred = linear_model.predict(X_test_scaled)

mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
rmse = np.sqrt(((y_test - y_pred) ** 2).mean())

print(f'MAE:  {mae:.2f}')
print(f'RMSE: {rmse:.2f}')
print(f'R2:   {r2:.3f}')

Stage 7: Iterate and Improve

Stage 8: Deploy the Model

import joblib

# Save the trained model
joblib.dump(linear_model, 'house_price_model.pkl')
print('Model saved.')

# Later, load and predict in production
loaded_model = joblib.load('house_price_model.pkl')
prediction = loaded_model.predict(X_test_scaled[:1])
print(f'Prediction: ${prediction[0]:,.0f}')

Stage 9: Monitor and Retrain

The Workflow as a Scikit-learn Pipeline

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression

# Chain preprocessing and model in one object
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('model', LinearRegression())
])

# fit() applies scaler then trains the model
pipeline.fit(X_train, y_train)

# predict() applies scaler then predicts
y_pred = pipeline.predict(X_test)
print('Pipeline prediction done.')

Quick Check

Lesson Recap