HTML mit BeautifulSoup parsen

BeautifulSoup installieren

# pip install beautifulsoup4 lxml
from bs4 import BeautifulSoup

html = "<h1>Hello</h1><p class='intro'>World</p>"
soup = BeautifulSoup(html, "lxml")
print(soup.h1.text)   # Hello

Elemente finden

from bs4 import BeautifulSoup

html = "<ul><li>A</li><li>B</li><li>C</li></ul>"
soup = BeautifulSoup(html, "lxml")
print(soup.find("li").text)       # A
print([li.text for li in soup.find_all("li")])  # [A,B,C]

CSS-Selektoren

from bs4 import BeautifulSoup

html = '<div class="card"><a href="/p/1">Link 1</a></div>'
soup = BeautifulSoup(html, "lxml")
print(soup.select("div.card a")[0]["href"])   # /p/1
print(soup.select_one("a")["href"])          # /p/1

Auf Attribute zugreifen

from bs4 import BeautifulSoup

html = '<a href="https://example.com" class="link external">Click</a>'
soup = BeautifulSoup(html, "lxml")
a = soup.find("a")
print(a["href"])           # https://example.com
print(a.get("class", [])) # ['link', 'external']

Parse-Baum durchlaufen

from bs4 import BeautifulSoup

html = "<div><h2>Title</h2><p>Para</p></div>"
soup = BeautifulSoup(html, "lxml")
h2 = soup.find("h2")
print(h2.parent.name)         # div
print(h2.next_sibling.text)   # Para (next sibling)

Text extrahieren

from bs4 import BeautifulSoup

html = "<div>  <p>Hello</p>  <p> World </p>  </div>"
soup = BeautifulSoup(html, "lxml")
print(soup.div.get_text(separator=" ", strip=True))
# Hello World

Scraping mit requests + BeautifulSoup

import requests
from bs4 import BeautifulSoup

url = "https://books.toscrape.com"
r = requests.get(url, timeout=10)
soup = BeautifulSoup(r.text, "lxml")

titles = [h3.find("a")["title"] for h3 in soup.select("article.product_pod h3")]
print(titles[:3])

Relative URLs verarbeiten

from urllib.parse import urljoin
base = "https://example.com/products/"
href = "../about"
print(urljoin(base, href))   # https://example.com/about

Zeichenkodierungen verarbeiten

import requests
from bs4 import BeautifulSoup

r = requests.get("https://example.com")
print(r.encoding)   # e.g. utf-8
soup = BeautifulSoup(r.content, "lxml", from_encoding=r.encoding)

Ändern und serialisieren

from bs4 import BeautifulSoup

html = "<p>Hello <b>world</b></p>"
soup = BeautifulSoup(html, "lxml")
soup.b.decompose()   # remove the <b> tag
print(str(soup.p))   # <p>Hello </p>

Grenzen von BeautifulSoup

# BeautifulSoup: good for static HTML
# Playwright: for JavaScript-rendered SPAs
# Scrapy: for large-scale crawls with pipelines

# pip install playwright
# playwright install
# from playwright.async_api import async_playwright

Schnelltest

Zusammenfassung