JavaScript와 스크래핑 방지 대책 처리

JavaScript가 중요한 이유

# Static scraper sees:
# <div id="app"></div>

# Rendered page has:
# <div id="app">
#   <h1>Product List</h1>
#   <ul>... dynamically loaded items ...</ul>
# </div>

Python용 Playwright

# pip install playwright
# playwright install
import asyncio
from playwright.async_api import async_playwright

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()
        await page.goto("https://example.com")
        title = await page.inner_text("h1")
        print(title)
        await browser.close()

asyncio.run(main())

요소 기다리기

from playwright.async_api import async_playwright
import asyncio

async def scrape():
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()
        await page.goto("https://spa-example.com")
        await page.wait_for_selector(".product-list")
        items = await page.query_selector_all(".product-item")
        texts = [await i.inner_text() for i in items]

Selenium

# pip install selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("https://example.com")
elem = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, "h1"))
)
print(elem.text)
driver.quit()

헤드리스 모드

from playwright.async_api import async_playwright
import asyncio

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)  # no window
        page = await browser.new_page()
        await page.goto("https://example.com")
        html = await page.content()
        await browser.close()

네트워크 요청 가로채기

from playwright.async_api import async_playwright
import asyncio

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()

        await page.route("**/*.{png,jpg,css}", lambda route: route.abort())
        await page.goto("https://example.com")   # loads 3x faster

요청 속도 제한과 지연

import time, random

def polite_scrape(urls):
    for url in urls:
        fetch(url)
        time.sleep(random.uniform(1, 3))  # 1-3 s random delay

사용자 에이전트 순환

import requests, random

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ...",
]

def fetch(url):
    return requests.get(url, headers={"User-Agent": random.choice(USER_AGENTS)})

쿠키와 세션

import requests

with requests.Session() as s:
    s.post("https://example.com/login", data={"user":"me","pass":"pw"})
    # session cookie kept automatically:
    dashboard = s.get("https://example.com/dashboard")

CAPTCHA와 법적 고려 사항

# Before scraping:
# 1. Check robots.txt: https://example.com/robots.txt
# 2. Review the Terms of Service for "scraping" or "automated access" clauses
# 3. Only scrape public data
# 4. Honour Crawl-delay directives
# 5. Cache responses to avoid repeated hits

Playwright 스크린샷 및 PDF

from playwright.async_api import async_playwright
import asyncio

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()
        await page.goto("https://example.com")
        await page.screenshot(path="page.png", full_page=True)
        await page.pdf(path="page.pdf")
        await browser.close()

빠른 확인

복습