الاستيفاء والتعويض المتقدم

متى يتفوق الاستيفاء على fillna()

import pandas as pd
import numpy as np

df = pd.DataFrame({
    'day': [1, 2, 3, 4, 5],
    'temp': [20.0, np.nan, np.nan, np.nan, 30.0]
})

# Linear interpolation fills gaps smoothly
df['temp_interp'] = df['temp'].interpolate(method='linear')
print(df)
#    day  temp  temp_interp
# 0    1  20.0         20.0
# 1    2   NaN         22.5
# 2    3   NaN         25.0
# 3    4   NaN         27.5
# 4    5  30.0         30.0

أساليب interpolate()

import pandas as pd
import numpy as np

s = pd.Series([0, np.nan, np.nan, 8.0])

print('linear:', s.interpolate('linear').tolist())
# [0.0, 2.6666..., 5.3333..., 8.0]

print('polynomial(2):', s.interpolate('polynomial', order=2).tolist())
# [0.0, 2.222..., 5.111..., 8.0]

الاستيفاء الزمني باستخدام DatetimeIndex

import pandas as pd
import numpy as np

# Unequal gaps: Jan 1, Jan 3, Jan 10
idx = pd.to_datetime(['2024-01-01', '2024-01-03', '2024-01-10'])
s = pd.Series([100.0, np.nan, 170.0], index=idx)

# linear treats gaps as equal (position-based)
print(s.interpolate('linear').tolist())  # [100.0, 135.0, 170.0]

# time accounts for actual day count
# Jan 1 to Jan 3 = 2 days, Jan 1 to Jan 10 = 9 days
# fraction: 2/9 of the way from 100 to 170
print(s.interpolate('time').tolist())   # [100.0, 115.55..., 170.0]

تحديد نطاق الاستيفاء

import pandas as pd
import numpy as np

s = pd.Series([1.0, np.nan, np.nan, np.nan, np.nan, 10.0])

# Only fill the first 2 NaN positions
filled = s.interpolate('linear', limit=2)
print(filled.tolist())
# [1.0, 2.8, 4.6, NaN, NaN, 10.0]

اختيار الإسناد بالمتوسط أم بالوسيط

import pandas as pd
import numpy as np

# Skewed income data with an outlier
income = pd.Series([30000, 35000, 32000, 1_000_000, np.nan])

print('Mean:  ', income.mean())    # ~274000 — pulled by outlier
print('Median:', income.median())  # ~33500 — robust choice

# Prefer median for skewed data
filled = income.fillna(income.median())
print(filled.tolist())
# [30000, 35000, 32000, 1000000, 33500.0]

مفهوم الإسناد باستخدام KNN

import pandas as pd
import numpy as np
from sklearn.impute import KNNImputer

df = pd.DataFrame({
    'age': [25, 35, np.nan, 45],
    'income': [50000, 70000, 60000, np.nan]
})

imputer = KNNImputer(n_neighbors=2)
df_imputed = pd.DataFrame(
    imputer.fit_transform(df),
    columns=df.columns
)
print(df_imputed.round(1))
#     age   income
# 0  25.0  50000.0
# 1  35.0  70000.0
# 2  30.0  60000.0   <- filled from neighbours
# 3  45.0  65000.0   <- filled from neighbours

الإسناد المتعدد باستخدام IterativeImputer

import pandas as pd
import numpy as np
from sklearn.experimental import enable_iterative_imputer  # noqa
from sklearn.impute import IterativeImputer

df = pd.DataFrame({
    'x1': [1, 2, np.nan, 4, 5],
    'x2': [2, np.nan, 6, 8, 10],
    'y':  [3, 5, 7, np.nan, 11]
})

imp = IterativeImputer(max_iter=10, random_state=0)
df_filled = pd.DataFrame(imp.fit_transform(df), columns=df.columns)
print(df_filled.round(2))

أعمدة المؤشرات للبيانات المفقودة

import pandas as pd
import numpy as np

df = pd.DataFrame({'salary': [50000, np.nan, 70000, np.nan, 90000]})

# Add indicator before filling
df['salary_was_missing'] = df['salary'].isna().astype(int)

# Then fill
df['salary'] = df['salary'].fillna(df['salary'].median())
print(df)
#    salary  salary_was_missing
# 0  50000.0                   0
# 1  70000.0                   1
# 2  70000.0                   0
# 3  70000.0                   1
# 4  90000.0                   0

الإسناد وتسرّب البيانات

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split

df = pd.DataFrame({'feature': [1, 2, np.nan, 4, np.nan, 6, 7, 8]})
train, test = train_test_split(df, test_size=0.25, random_state=0)

# Fit ONLY on training data
imp = SimpleImputer(strategy='mean')
train['feature'] = imp.fit_transform(train[['feature']])

# Transform test using training statistics
test['feature'] = imp.transform(test[['feature']])
print('Train mean used:', imp.statistics_[0])

مقارنة استراتيجيات الإسناد بصريًا

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(0)
original = np.random.exponential(scale=5, size=200)
with_nan = original.copy()
with_nan[np.random.choice(200, 40, replace=False)] = np.nan

s = pd.Series(with_nan)

fig, axes = plt.subplots(1, 2, figsize=(10, 4))
s.dropna().hist(ax=axes[0], bins=20, title='Original (no NaN)')
s.fillna(s.mean()).hist(ax=axes[1], bins=20, title='Mean-imputed')
plt.tight_layout()
plt.savefig('imputation_comparison.png')
print('Saved comparison chart')

اختيار استراتيجية الإسناد المناسبة

# Decision guide (no runnable code)
#
# Missing < 5%  AND data is MCAR?  -> Mean/median fill is fine
# Time series with stable trend?   -> ffill() with limit
# Features are correlated?         -> KNNImputer or IterativeImputer
# Categorical column?              -> Mode fill or 'Unknown' sentinel
# Going into ML model?             -> Add indicator column + fill
# Research / publication quality?  -> Multiple imputation (MICE)
print('Strategy selected based on context')

تحقق سريع

مراجعة الدرس