BeautifulSoup으로 HTML 파싱하기

에이전트에서 HTML을 구문 분석하는 이유

BeautifulSoup 객체 만들기

from bs4 import BeautifulSoup
import httpx

# Fetch HTML
response = httpx.get('https://example.com', timeout=10.0)
html = response.text

# Parse it
soup = BeautifulSoup(html, 'html.parser')

# Get the page title
print(soup.title.text)  # 'Example Domain'

find() — 단일 요소 찾기

from bs4 import BeautifulSoup

html = '<div class="content"><h1>Title</h1><p>Body text here.</p></div>'
soup = BeautifulSoup(html, 'html.parser')

# Find by tag + class
content_div = soup.find('div', class_='content')
if content_div:
    heading = content_div.find('h1')
    print(heading.text)  # 'Title'

# Find by id
sidebar = soup.find('div', id='sidebar')  # None if absent

find_all() — 여러 요소 찾기

from bs4 import BeautifulSoup

html = '<ul><li>Apple</li><li>Banana</li><li>Cherry</li></ul>'
soup = BeautifulSoup(html, 'html.parser')

items = soup.find_all('li')
for item in items:
    print(item.text)  # Apple, Banana, Cherry

# Limit results
first_two = soup.find_all('li', limit=2)

select()를 사용한 CSS 선택자

from bs4 import BeautifulSoup

html = '''
<table>
  <tr><td class="name">Alice</td><td class="score">95</td></tr>
  <tr><td class="name">Bob</td><td class="score">87</td></tr>
</table>
'''
soup = BeautifulSoup(html, 'html.parser')

# Select all td elements inside tr inside table
cells = soup.select('table tr td')
for cell in cells:
    print(cell.text)

# Select only name cells
names = soup.select('td.name')
for n in names:
    print(n.text)  # Alice, Bob

.text와 .strip()으로 텍스트 추출하기

from bs4 import BeautifulSoup

html = '<p>  \n  Price: <strong>$29.99</strong>  \n  </p>'
soup = BeautifulSoup(html, 'html.parser')

paragraph = soup.find('p')

# .text includes nested tag content
print(paragraph.text)          # '  \n  Price: $29.99  \n  '
print(paragraph.text.strip())  # 'Price: $29.99'

# get_text with separator
print(paragraph.get_text(separator=' ', strip=True))  # 'Price: $29.99'

.get()으로 속성 추출하기

from bs4 import BeautifulSoup

html = '''
<a href="https://example.com/page" data-id="42">Click here</a>
<img src="/images/logo.png" alt="Logo">
'''
soup = BeautifulSoup(html, 'html.parser')

link = soup.find('a')
print(link.get('href'))     # 'https://example.com/page'
print(link.get('data-id'))  # '42'
print(link.get('class'))    # None (no class attribute)

img = soup.find('img')
print(img.get('src'))       # '/images/logo.png'

구문 분석 트리 탐색하기

from bs4 import BeautifulSoup

html = '''
<div class="article">
  <h2>Headline</h2>
  <p>First paragraph.</p>
  <p>Second paragraph.</p>
</div>
'''
soup = BeautifulSoup(html, 'html.parser')

h2 = soup.find('h2')
print(h2.text)                    # 'Headline'
print(h2.parent['class'])         # ['article']
print(h2.next_sibling.next_sibling.text)  # 'First paragraph.'

페이지에서 모든 링크 추출하기

from bs4 import BeautifulSoup
import httpx

def extract_links(url: str) -> list:
    response = httpx.get(url, timeout=10.0)
    soup = BeautifulSoup(response.text, 'html.parser')

    links = []
    for tag in soup.find_all('a'):
        href = tag.get('href')
        if href and href.startswith('http'):
            links.append(href)
    return links

# urls = extract_links('https://news.ycombinator.com')
# print(urls[:5])

잘못된 HTML을 안정적으로 처리하기

from bs4 import BeautifulSoup

# Broken HTML — missing closing tags
html = '<div><p>Hello <strong>World</div>'
soup = BeautifulSoup(html, 'html.parser')

# BS4 repairs the tree automatically
print(soup.prettify())
# <div><p>Hello <strong>World</strong></p></div>

# Safe chained access
price = soup.find('span', class_='price')
price_text = price.text.strip() if price else 'N/A'
print(price_text)  # 'N/A'

완전한 에이전트 스크레이퍼 도구

from bs4 import BeautifulSoup
import httpx

def scrape_article(url: str) -> dict:
    response = httpx.get(url, headers={'User-Agent': 'MyAgent/1.0'}, timeout=10.0)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'html.parser')

    title = soup.find('h1')
    paragraphs = soup.find_all('p')

    return {
        'url': url,
        'title': title.text.strip() if title else '',
        'text': ' '.join(p.text.strip() for p in paragraphs[:5]),
        'links': [a.get('href') for a in soup.find_all('a', href=True)][:10]
    }

지식 확인: BeautifulSoup

복습: BeautifulSoup으로 HTML 파싱하기