timeitとmemory_profilerによるプロファイリング

最適化の前にプロファイリングする理由

import pandas as pd
import numpy as np
import timeit

# Naive approach vs vectorised approach — which is faster?
np.random.seed(0)
df = pd.DataFrame({'a': np.random.randn(100000), 'b': np.random.randn(100000)})

t1 = timeit.timeit(lambda: df['a'] + df['b'], number=100)
t2 = timeit.timeit(lambda: [a + b for a, b in zip(df['a'], df['b'])], number=10)

print(f'Vectorised (100 runs): {t1:.3f}s')
print(f'List comprehension (10 runs): {t2:.3f}s')
print(f'Per-run speedup: ~{(t2/10)/(t1/100):.0f}x')

timeit：Python組み込みのタイマー

import timeit
import pandas as pd
import numpy as np

np.random.seed(0)
df = pd.DataFrame({'x': np.random.randn(10000)})

# Compare three ways to square a column
time_a = timeit.timeit(lambda: df['x'] ** 2, number=1000)
time_b = timeit.timeit(lambda: df['x'].apply(lambda v: v**2), number=100)
time_c = timeit.timeit(lambda: np.square(df['x']), number=1000)

print(f'Operator **2  per run: {time_a/1000*1000:.3f} ms')
print(f'.apply(v**2) per run: {time_b/100*1000:.3f} ms')
print(f'np.square()  per run: {time_c/1000*1000:.3f} ms')

Jupyterの%timeitマジック

# In Jupyter notebook:
# %timeit df['x'] ** 2
# Output: 47.3 us +- 1.2 us per loop (mean +- std. dev. of 7 runs, 10000 loops each)

# Multi-line cell timing:
# %%timeit
# result = df['x'] ** 2
# total = result.sum()

# In a regular Python script, use timeit.timeit() instead:
import timeit
import pandas as pd, numpy as np
df = pd.DataFrame({'x': np.random.randn(10000)})
result = timeit.repeat(lambda: df['x']**2, number=1000, repeat=7)
print(f'Best run: {min(result)/1000*1000:.3f} ms per call')

関数単位のプロファイリングに使うcProfile

import cProfile
import pandas as pd
import numpy as np

np.random.seed(0)
df = pd.DataFrame({'a': np.random.randn(50000), 'b': np.random.randn(50000)})

def slow_pipeline(df):
    result = []
    for _, row in df.iterrows():
        result.append(row['a'] * row['b'])
    return pd.Series(result)

# Profile the function
print('Profile output (top 5 functions by cumulative time):')
cProfile.run('slow_pipeline(df.head(1000))', sort='cumulative')

メモリ使用量の測定

import pandas as pd
import numpy as np

np.random.seed(0)
df = pd.DataFrame({
    'id': range(100000),
    'name': ['user_' + str(i) for i in range(100000)],
    'score': np.random.randn(100000),
    'category': np.random.choice(['A', 'B', 'C', 'D'], 100000)
})

mem = df.memory_usage(deep=True)
print('Memory per column (bytes):')
print(mem)
print(f'\nTotal: {mem.sum() / 1e6:.2f} MB')

行単位のメモリ測定に使うmemory_profiler

# Install: pip install memory-profiler
# Usage as a decorator:

# from memory_profiler import profile
# @profile
# def process_data():
#     import pandas as pd, numpy as np
#     df = pd.DataFrame({'x': np.random.randn(1000000)})
#     result = df['x'].rolling(100).mean()    # <-- sees if this spikes RAM
#     return result

# Run: python -m memory_profiler script.py

# In Jupyter (after %load_ext memory_profiler):
# %memit pd.DataFrame({'x': np.random.randn(1000000)}).rolling(100).mean()

print('memory_profiler decorates functions for line-by-line RAM measurement.')
print('Typical output: Line # Mem usage Increment Line Contents')

オブジェクトにおけるsys.getsizeofとmemory_usageの比較

import sys
import pandas as pd
import numpy as np

df = pd.DataFrame({'x': np.random.randn(100000)})

# sys.getsizeof reports only the container metadata — misleading!
print('sys.getsizeof(df):', sys.getsizeof(df), 'bytes (misleading!)')

# memory_usage gives the true memory
true_mem = df.memory_usage(deep=True).sum()
print('df.memory_usage(deep=True).sum():', true_mem, 'bytes')
print(f'True memory: {true_mem/1e6:.2f} MB')

タイミング用コンテキストマネージャーパターン

import time
import contextlib
import pandas as pd
import numpy as np

@contextlib.contextmanager
def timer(name):
    start = time.perf_counter()
    yield
    elapsed = time.perf_counter() - start
    print(f'[{name}] {elapsed*1000:.1f} ms')

np.random.seed(0)
df = pd.DataFrame({'x': np.random.randn(500000), 'y': np.random.randn(500000)})

with timer('rolling mean'):
    result1 = df['x'].rolling(30).mean()

with timer('groupby mean'):
    df['group'] = df['x'].round(0)
    result2 = df.groupby('group')['y'].mean()

Pandas操作を体系的にプロファイリングする

import pandas as pd
import numpy as np
import timeit

np.random.seed(0)
df = pd.DataFrame({
    'value': np.random.randn(100000),
    'category': np.random.choice(['A', 'B', 'C', 'D', 'E'], 100000)
})

# Benchmark two groupby implementations
t1 = timeit.timeit(lambda: df.groupby('category')['value'].mean(), number=100)
df2 = df.copy()
df2['category'] = df2['category'].astype('category')
t2 = timeit.timeit(lambda: df2.groupby('category')['value'].mean(), number=100)

print(f'Object dtype groupby: {t1/100*1000:.2f} ms per call')
print(f'Categorical dtype:    {t2/100*1000:.2f} ms per call')
print(f'Speedup: {t1/t2:.1f}x')

プロファイリングを行うタイミング: 実用的な目安

スナップショット: プロファイリングツールの概要

クイックチェック

レッスンのまとめ