배치 정규화: 안정적이고 빠른 학습

# Without batch norm: deep networks train slowly and
# require very careful weight init and LR tuning.

# With batch norm: can use higher learning rates,
# less sensitive to initialisation, acts as regulariser.

# Batch norm normalises each feature to:
# mean=0, std=1 within the batch, then
# applies learnable scale (gamma) and shift (beta).
print('Batch Norm: normalize -> scale -> shift')

import torch

def batch_norm_manual(x, gamma, beta, eps=1e-5):
    # x shape: (batch_size, features)
    mu = x.mean(dim=0)           # mean per feature
    var = x.var(dim=0, unbiased=False)  # var per feature
    x_norm = (x - mu) / (var + eps).sqrt()
    return gamma * x_norm + beta  # scale and shift

x = torch.randn(32, 8)  # batch=32, 8 features
gamma = torch.ones(8)
beta = torch.zeros(8)

out = batch_norm_manual(x, gamma, beta)
print('Mean near 0:', out.mean(dim=0).abs().max().item() < 0.01)
print('Std near 1:', (out.std(dim=0) - 1).abs().max().item() < 0.01)

import torch
import torch.nn as nn

model = nn.Sequential(
    nn.Linear(16, 64),
    nn.BatchNorm1d(64),   # after linear, before activation
    nn.ReLU(),
    nn.Linear(64, 32),
    nn.BatchNorm1d(32),
    nn.ReLU(),
    nn.Linear(32, 10)
)

x = torch.randn(32, 16)   # batch of 32
out = model(x)
print(out.shape)           # torch.Size([32, 10])

import torch
import torch.nn as nn

bn = nn.BatchNorm1d(4)

# Training mode: uses batch statistics, updates running stats
bn.train()
x = torch.randn(8, 4)
out_train = bn(x)

# After training, running_mean and running_var are populated
print('Running mean:', bn.running_mean)

# Eval mode: uses running statistics (deterministic)
bn.eval()
x_new = torch.randn(1, 4)   # single sample -- works!
out_eval = bn(x_new)
print(out_eval.shape)         # torch.Size([1, 4])

import torch.nn as nn

bn = nn.BatchNorm1d(8)
print('gamma (weight):', bn.weight.data)   # all 1s
print('beta (bias):', bn.bias.data)         # all 0s
print('gamma requires_grad:', bn.weight.requires_grad)  # True
print('beta requires_grad:', bn.bias.requires_grad)     # True

# Total trainable params in this BN layer:
# 2 * 8 = 16 (gamma and beta for 8 features)
params = sum(p.numel() for p in bn.parameters())
print('Params:', params)   # 16

import torch
import torch.nn as nn
import torch.optim as optim

def make_model(use_bn):
    layers = [nn.Linear(16, 64)]
    if use_bn: layers.append(nn.BatchNorm1d(64))
    layers.append(nn.ReLU())
    layers.append(nn.Linear(64, 2))
    return nn.Sequential(*layers)

X = torch.randn(200, 16)
y = torch.randint(0, 2, (200,))

for use_bn in [False, True]:
    model = make_model(use_bn)
    opt = optim.SGD(model.parameters(), lr=0.1)
    crit = nn.CrossEntropyLoss()
    for _ in range(20):
        opt.zero_grad(); loss = crit(model(X), y)
        loss.backward(); opt.step()
    print(f'BN={use_bn}: final_loss={loss.item():.4f}')

# Key insight: batch norm introduces noise proportional to
# 1/sqrt(batch_size) because batch statistics are noisy
# estimates of population statistics.

# Small batch (e.g., 8): high noise -> more regularisation
# Large batch (e.g., 512): low noise -> less regularisation

# Common pattern: use batch norm AND a small dropout
# for strong regularisation in deep networks
model = __import__('torch').nn.Sequential(
    __import__('torch').nn.Linear(32, 128),
    __import__('torch').nn.BatchNorm1d(128),
    __import__('torch').nn.ReLU(),
    __import__('torch').nn.Dropout(0.2)  # mild dropout
)
print('BN + light Dropout: balanced regularisation')

import torch
import torch.nn as nn

# Standard CNN block: Conv -> BN -> ReLU
conv_block = nn.Sequential(
    nn.Conv2d(3, 64, kernel_size=3, padding=1),
    nn.BatchNorm2d(64),    # 64 = number of output channels
    nn.ReLU(inplace=True)
)

# Input: batch of 8 RGB images, 32x32 pixels
x = torch.randn(8, 3, 32, 32)
out = conv_block(x)
print(out.shape)   # torch.Size([8, 64, 32, 32])

import torch
import torch.nn as nn

x = torch.randn(4, 8)  # batch=4, features=8

# Batch Norm: normalise across batch for each feature
bn = nn.BatchNorm1d(8)
bn_out = bn(x)   # statistics computed over 4 samples

# Layer Norm: normalise across features for each sample
ln = nn.LayerNorm(8)
ln_out = ln(x)   # statistics computed over 8 features

print('BN output shape:', bn_out.shape)   # (4, 8)
print('LN output shape:', ln_out.shape)   # (4, 8)
# Same shape, different normalisation axes

import torch.nn as nn

def freeze_bn(module):
    '''Keep BN in eval mode during fine-tuning.'''
    if isinstance(module, (nn.BatchNorm1d,
                           nn.BatchNorm2d,
                           nn.BatchNorm3d)):
        module.eval()  # use running stats, not batch stats
        module.weight.requires_grad_(False)
        module.bias.requires_grad_(False)

model = nn.Sequential(
    nn.Linear(4, 8),
    nn.BatchNorm1d(8),
    nn.ReLU()
)
model.apply(freeze_bn)
print('BN frozen for fine-tuning')

import torch
import torch.nn as nn

x = torch.randn(4, 16, 10)  # (batch, channels, seq_len)

# GroupNorm: 4 groups of 4 channels each
gn = nn.GroupNorm(num_groups=4, num_channels=16)
print('GroupNorm:', gn(x).shape)

# InstanceNorm: normalise each sample+channel independently
ins = nn.InstanceNorm1d(16)
print('InstanceNorm:', ins(x).shape)

# LayerNorm: normalise across last N dimensions
ln = nn.LayerNorm([16, 10])
print('LayerNorm:', ln(x).shape)