이상값 감지와 처리

import pandas as pd
import numpy as np

df = pd.read_parquet('sales_clean.parquet')
print(df['revenue'].describe())

import matplotlib.pyplot as plt
import seaborn as sns

fig, ax = plt.subplots(figsize=(6, 4))
df['revenue'].plot(kind='box', ax=ax)
ax.set_title('Revenue Distribution — Box Plot')
plt.tight_layout()
plt.show()

Q1 = df['revenue'].quantile(0.25)
Q3 = df['revenue'].quantile(0.75)
IQR = Q3 - Q1

lower = Q1 - 1.5 * IQR
upper = Q3 + 1.5 * IQR

outliers = df[(df['revenue'] < lower) | (df['revenue'] > upper)]
print(f'Q1={Q1:.0f}, Q3={Q3:.0f}, IQR={IQR:.0f}')
print(f'Bounds: [{lower:.0f}, {upper:.0f}]')
print(f'Outliers: {len(outliers)}')

mean = df['revenue'].mean()
std = df['revenue'].std()

df['revenue_z'] = (df['revenue'] - mean) / std
z_outliers = df[df['revenue_z'].abs() > 3]

print(f'Z-score outliers (|z|>3): {len(z_outliers)}')
print(z_outliers[['revenue', 'revenue_z']].head())

df['is_revenue_outlier'] = (df['revenue'] < lower) | (df['revenue'] > upper)

print('Flagged rows:', df['is_revenue_outlier'].sum())
print(df.groupby('is_revenue_outlier')['revenue'].describe())

df['revenue_capped'] = df['revenue'].clip(lower=lower, upper=upper)

print('Original max:', df['revenue'].max())
print('Capped max:', df['revenue_capped'].max())
print('Rows changed:', (df['revenue'] != df['revenue_capped']).sum())

df['log_revenue'] = np.log1p(df['revenue'])

fig, axes = plt.subplots(1, 2, figsize=(12, 4))
df['revenue'].hist(bins=50, ax=axes[0])
axes[0].set_title('Original Revenue')
df['log_revenue'].hist(bins=50, ax=axes[1])
axes[1].set_title('Log Revenue')
plt.tight_layout()
plt.show()

numeric_cols = df.select_dtypes(include=['number']).columns

outlier_report = {}
for col in numeric_cols:
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    n_out = ((df[col] < Q1 - 1.5*IQR) | (df[col] > Q3 + 1.5*IQR)).sum()
    outlier_report[col] = n_out

print(pd.Series(outlier_report).sort_values(ascending=False))

fig, ax = plt.subplots(figsize=(8, 5))
df.plot.scatter(x='quantity', y='unit_price', alpha=0.3, ax=ax)
ax.set_title('Quantity vs. Unit Price — Bivariate Outliers')
plt.tight_layout()
plt.show()

outlier_log = {
    'column': 'revenue',
    'method': 'IQR 1.5x',
    'lower_bound': round(lower, 2),
    'upper_bound': round(upper, 2),
    'rows_flagged': int(df['is_revenue_outlier'].sum()),
    'treatment': 'cap (winsorise)'
}
for k, v in outlier_log.items():
    print(f'{k}: {v}')

cols_to_save = [c for c in df.columns if c not in ['revenue_z']]
df[cols_to_save].to_parquet('sales_treated.parquet', index=False)
print('Outlier-treated dataset saved:', df.shape)