기술 통계와 정규성 검정

기술 통계와 추론 통계 비교

확장된 기술 통계

import pandas as pd
from scipy import stats
import numpy as np

np.random.seed(0)
data = np.concatenate([
    np.random.exponential(scale=2, size=500),  # right-skewed
    np.random.normal(loc=5, scale=1, size=500)
])

print('Mean:', round(data.mean(), 3))
print('Std:', round(data.std(), 3))
print('Skewness:', round(stats.skew(data), 3))
print('Kurtosis (excess):', round(stats.kurtosis(data), 3))

왜도 이해하기

import numpy as np
from scipy import stats

# Right-skewed: income-like data
right_skewed = np.random.lognormal(mean=1, sigma=1, size=1000)
print('Right skew:', round(stats.skew(right_skewed), 3))

# After log transform
print('After log transform:', round(stats.skew(np.log(right_skewed)), 3))

# Symmetric normal
normal_data = np.random.normal(0, 1, 1000)
print('Normal skew:', round(stats.skew(normal_data), 3))

정규성 검정이 중요한 이유

Shapiro-Wilk 검정

import numpy as np
from scipy import stats

np.random.seed(42)

# Normal data
normal = np.random.normal(0, 1, 100)
stat, p = stats.shapiro(normal)
print(f'Normal data: W={stat:.4f}, p={p:.4f}')
print('Conclusion:', 'Looks normal' if p > 0.05 else 'Not normal')

# Skewed data
skewed = np.random.exponential(1, 100)
stat2, p2 = stats.shapiro(skewed)
print(f'Skewed data: W={stat2:.4f}, p={p2:.4f}')
print('Conclusion:', 'Looks normal' if p2 > 0.05 else 'Not normal')

Kolmogorov-Smirnov 검정

import numpy as np
from scipy import stats

np.random.seed(0)
data = np.random.normal(loc=5, scale=2, size=200)

# One-sample K-S test against N(5, 2) distribution
stat, p = stats.kstest(data, 'norm', args=(5, 2))
print(f'K-S test: stat={stat:.4f}, p={p:.4f}')
print('Follows N(5,2)?', 'Yes' if p > 0.05 else 'No')

# Two-sample K-S test
data2 = np.random.normal(loc=5.5, scale=2, size=200)
stat2, p2 = stats.ks_2samp(data, data2)
print(f'Two-sample K-S: stat={stat2:.4f}, p={p2:.4f}')

시각적 정규성 확인: QQ 플롯

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

np.random.seed(1)
data = np.random.exponential(2, 200)

# Q-Q plot
(osm, osr), (slope, intercept, r) = stats.probplot(data, dist='norm')
print(f'R-squared of Q-Q fit: {r**2:.4f}')  # Close to 1 = normal
# Visual inspection: if plotting, points on the line = normal
# plt.figure()
# stats.probplot(data, dist='norm', plot=plt)
# plt.show()

실전에서의 중심 극한 정리

import numpy as np
from scipy import stats

np.random.seed(0)
# Population: heavily right-skewed (exponential)
population = np.random.exponential(scale=1, size=10000)
print('Population skewness:', round(stats.skew(population), 3))

# Sample means are approximately normal (CLT)
sample_means = [np.random.choice(population, 50).mean()
                for _ in range(1000)]
print('Sample means skewness:', round(stats.skew(sample_means), 3))
_, p = stats.shapiro(sample_means)
print('Shapiro p-value for means:', round(p, 4))

그룹별 정규성

import pandas as pd
import numpy as np
from scipy import stats

np.random.seed(0)
df = pd.DataFrame({
    'region': ['N']*50 + ['S']*50 + ['E']*50,
    'sales': np.concatenate([
        np.random.normal(100, 20, 50),
        np.random.normal(110, 25, 50),
        np.random.normal(95, 15, 50)
    ])
})

for region, group in df.groupby('region'):
    stat, p = stats.shapiro(group['sales'])
    print(f'Region {region}: W={stat:.4f}, p={p:.4f} -> '
          f'{"Normal" if p>0.05 else "Not normal"}')

비모수 대안

import numpy as np
from scipy import stats

np.random.seed(0)
# Non-normal data
group_a = np.random.exponential(2, 40)
group_b = np.random.exponential(2.5, 40)

# Parametric would be wrong here; use non-parametric
stat, p = stats.mannwhitneyu(group_a, group_b, alternative='two-sided')
print(f'Mann-Whitney U test: U={stat:.1f}, p={p:.4f}')
print('Groups differ?' , 'Yes' if p < 0.05 else 'No')

여러 열의 통계 요약

import pandas as pd
import numpy as np
from scipy import stats

np.random.seed(0)
df = pd.DataFrame({
    'age': np.random.normal(35, 10, 200),
    'income': np.random.lognormal(10, 1, 200),
    'score': np.random.uniform(0, 100, 200)
})

rows = []
for col in df.select_dtypes(include='number').columns:
    s = stats.skew(df[col])
    _, p = stats.shapiro(df[col][:200])
    rows.append({'column': col, 'skewness': round(s, 3),
                 'shapiro_p': round(p, 4), 'normal': p > 0.05})

print(pd.DataFrame(rows).to_string(index=False))

빠른 확인

수업 요약