Lidando com paginação e conteúdo dinâmico

O problema da paginação

Detectando links para a próxima página

from bs4 import BeautifulSoup
import httpx

def scrape_all_pages(start_url: str) -> list:
    results = []
    url = start_url

    while url:
        response = httpx.get(url, timeout=10.0)
        soup = BeautifulSoup(response.text, 'html.parser')

        # Collect data from this page
        for item in soup.select('.result-item'):
            results.append(item.text.strip())

        # Find the next page link
        next_link = soup.find('a', rel='next') or soup.find('a', string='Next')
        url = next_link.get('href') if next_link else None

    return results

Padrões de URL com número de página

import httpx

def scrape_paginated_api(base_url: str, max_pages: int = 10) -> list:
    all_items = []

    for page in range(1, max_pages + 1):
        response = httpx.get(
            base_url,
            params={'page': page, 'per_page': 50},
            timeout=10.0
        )
        response.raise_for_status()
        data = response.json()

        items = data.get('results', [])
        if not items:
            break  # No more data

        all_items.extend(items)
        print(f'Page {page}: fetched {len(items)} items')

    return all_items

Paginação baseada em cursor

import httpx

def fetch_with_cursor(url: str, headers: dict) -> list:
    all_data = []
    cursor = None

    while True:
        params = {'cursor': cursor} if cursor else {}
        response = httpx.get(url, headers=headers, params=params, timeout=10.0)
        response.raise_for_status()
        data = response.json()

        all_data.extend(data.get('items', []))

        cursor = data.get('next_cursor')  # None when done
        if not cursor:
            break

    return all_data

Conteúdo estático vs. dinâmico

Instalando e importando Playwright

# Installation (run in terminal):
# pip install playwright
# playwright install chromium

from playwright.async_api import async_playwright
import asyncio

async def fetch_dynamic(url: str) -> str:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto(url)
        html = await page.content()  # Full rendered HTML
        await browser.close()
    return html

# html = asyncio.run(fetch_dynamic('https://example.com'))

Aguardando o carregamento do conteúdo dinâmico

from playwright.async_api import async_playwright

async def scrape_dynamic_table(url: str) -> list:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto(url)

        # Wait for the table to appear in the DOM
        await page.wait_for_selector('table.results', timeout=10000)

        html = await page.content()
        await browser.close()

    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    return [td.text for td in soup.select('table.results td')]

Clicando e interagindo com páginas

from playwright.async_api import async_playwright

async def click_load_more(url: str) -> str:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto(url)

        # Click the 'Load More' button
        load_more = page.locator('button:text("Load More")')
        if await load_more.count() > 0:
            await load_more.click()
            await page.wait_for_timeout(2000)  # wait 2s for content

        html = await page.content()
        await browser.close()
    return html

Lidando com rolagem infinita

from playwright.async_api import async_playwright

async def scrape_infinite_scroll(url: str, scroll_count: int = 5) -> str:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto(url)

        for _ in range(scroll_count):
            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
            await page.wait_for_timeout(1500)  # wait for new content

        html = await page.content()
        await browser.close()
    return html

Combinando Playwright com BeautifulSoup

from playwright.async_api import async_playwright
from bs4 import BeautifulSoup

async def extract_js_rendered_data(url: str) -> list:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        page = await browser.new_page()
        await page.goto(url)
        await page.wait_for_selector('.product-card')
        html = await page.content()
        await browser.close()

    soup = BeautifulSoup(html, 'html.parser')
    products = []
    for card in soup.select('.product-card'):
        products.append({
            'name': card.select_one('.name').text.strip(),
            'price': card.select_one('.price').text.strip()
        })
    return products

Escolhendo a ferramenta certa

Verificação de conhecimentos: paginação e conteúdo dinâmico

Recapitulação: paginação e conteúdo dinâmico