Paginierung und mehrere Seiten verarbeiten

Paginierung verstehen

library(rvest)

# Common URL pagination patterns:
# Pattern 1: query parameter
# https://site.com/products?page=1
# https://site.com/products?page=2

# Pattern 2: path segment
# https://site.com/products/page/1/
# https://site.com/products/page/2/

# Build URLs for pages 1 to 5
base_url <- 'https://books.toscrape.com/catalogue/page-'
page_urls <- paste0(base_url, 1:5, '.html')
page_urls
# [1] 'https://books.toscrape.com/catalogue/page-1.html'
# [2] 'https://books.toscrape.com/catalogue/page-2.html' ...

URL-Muster mit sprintf erstellen

# Various URL building approaches

# paste0 for simple concatenation
pages <- 1:5
urls_v1 <- paste0('https://site.com/list?p=', pages)
urls_v1[1]  # 'https://site.com/list?p=1'

# sprintf for formatted strings
urls_v2 <- sprintf('https://site.com/page/%02d/', pages)
urls_v2[1]  # 'https://site.com/page/01/'

# With multiple parameters
urls_v3 <- sprintf('https://site.com/items?page=%d&size=20', pages)
urls_v3[3]  # 'https://site.com/items?page=3&size=20'

# glue package alternative
# glue('https://site.com/page/{pages}')

Mehrere Seiten mit map() auslesen

library(rvest)
library(purrr)

# Define a function to scrape one page
scrape_page <- function(url) {
  # In real code: page <- read_html(url)
  # Here we simulate with local HTML
  page <- read_html(sprintf(
    '<ul><li class="item">Item %d-A</li><li class="item">Item %d-B</li></ul>',
    url, url
  ))
  data.frame(
    title = html_text2(html_elements(page, '.item')),
    page  = url
  )
}

# Apply over pages 1 and 2 (simulated with numbers)
results <- map(1:2, scrape_page)
do.call(rbind, results)

Höfliches Scraping: Sys.sleep()

library(rvest)

# Polite scraping with delays between requests
scrape_with_delay <- function(urls, delay = 1.5) {
  results <- vector('list', length(urls))
  for (i in seq_along(urls)) {
    cat(sprintf('Scraping page %d of %d\n', i, length(urls)))
    # results[[i]] <- read_html(urls[[i]])
    Sys.sleep(delay)  # Wait before next request
  }
  results
}

# Also: randomize delay to appear more human-like
random_delay <- function(min = 1, max = 3) {
  Sys.sleep(runif(1, min = min, max = max))
}
random_delay()
cat('Always be polite to servers you scrape')

Einen User-Agent festlegen

library(rvest)

# Default user agent reveals R/rvest
# Better: set a descriptive, honest UA
ua <- paste0(
  'MyResearchBot/1.0 ',
  '(Academic research; ',
  'contact: researcher@university.edu)'
)

# With httr/httr2 for UA control:
# library(httr2)
# response <- request(url) |>
#   req_headers('User-Agent' = ua) |>
#   req_perform()
# html <- read_html(resp_body_string(response))

# Or use polite package for full compliance:
# library(polite)
# session <- bow('https://example.com', user_agent = ua)
cat('Honest UA: better for everyone')

robots.txt beachten

library(rvest)
# library(robotstxt)  # uncomment to use

# Check if scraping is allowed:
# rtxt <- robotstxt(domain = 'books.toscrape.com')
# rtxt$check(paths = '/catalogue/', bot = '*')
# TRUE means allowed

# Robots.txt typically looks like:
# User-agent: *
# Disallow: /admin/
# Disallow: /private/
# Allow: /catalogue/
# Crawl-delay: 2

# The polite package combines UA + robots.txt + delays:
# library(polite)
# session <- bow('https://books.toscrape.com')
# page <- scrape(session)
cat('Always check robots.txt before large-scale scraping')

Paginierungslinks folgen

library(rvest)

# Pattern: follow 'next' links dynamically
scrape_all_pages <- function(start_url, max_pages = 10) {
  all_data <- list()
  url <- start_url
  i <- 1
  while (!is.null(url) && i <= max_pages) {
    # page <- read_html(url)
    # data <- extract_data(page)
    # all_data[[i]] <- data
    # next_link <- html_attr(html_element(page, 'li.next a'), 'href')
    # url <- if (!is.na(next_link)) xml2::url_absolute(next_link, url)
    #        else NULL
    cat(sprintf('Would scrape page %d: %s\n', i, url))
    url <- NULL  # Stop in this demo
    i <- i + 1
  }
  do.call(rbind, all_data)
}
scrape_all_pages('https://books.toscrape.com')

session_follow_link() mit rvest

library(rvest)
# Stateful session with rvest
# s <- session('https://example.com/login')
# s <- session_submit(s,
#   html_form(read_html(s))[[1]],
#   list(username='me', password='pw')
# )
# After login, navigate:
# page2 <- session_follow_link(s, 'Products')
# page3 <- session_back(page2)   # go back
# session_history(page2)         # show history

# For simple cases, follow href directly:
# href <- html_attr(html_element(page, 'a.next'), 'href')
# next_page <- session_jump_to(s, href)
cat('session() maintains cookies across requests')

Die letzte Seite erkennen

library(rvest)
# Strategy 1: Check if 'Next' link exists
has_next <- function(page) {
  next_link <- html_element(page, 'li.next a')
  !is.na(html_attr(next_link, 'href'))
}

# Strategy 2: Compare item count to expected page size
page_size <- 20
check_done <- function(items_on_page) {
  items_on_page < page_size
}

# Strategy 3: Read total from pagination text
# e.g., 'Page 1 of 50' -> extract 50
extract_total_pages <- function(page) {
  text <- html_text2(html_element(page, '.current'))
  as.integer(gsub('.*of (\\d+).*', '\\1', text))
}
cat('Always plan your stopping condition')

Scraping-Fehler behandeln

library(rvest)

safe_scrape <- function(url) {
  tryCatch({
    # page <- read_html(url)
    # data <- extract_data(page)
    # return(data)
    cat(sprintf('Scraping: %s\n', url))
    data.frame(url = url, status = 'ok')
  }, error = function(e) {
    cat(sprintf('FAILED: %s -> %s\n', url, e$message))
    data.frame(url = url, status = 'error')
  })
}

urls <- c('https://example.com/page/1',
          'https://example.com/page/2')
results <- lapply(urls, safe_scrape)
do.call(rbind, results)

Ausgelesene Seiten zwischenspeichern

library(rvest)

scrape_cached <- function(url, cache_dir = 'cache') {
  dir.create(cache_dir, showWarnings = FALSE)
  # Create a filename from URL hash
  fname <- file.path(cache_dir,
    paste0(digest::digest(url), '.html'))
  if (file.exists(fname)) {
    cat('Loading from cache\n')
    return(read_html(fname))
  }
  cat('Downloading...\n')
  # page <- read_html(url)
  # xml2::write_html(page, fname)
  # return(page)
  cat('Cached to:', fname, '\n')
}

# Without digest: use URLencode or gsub
url <- 'https://example.com/page/1'
fname <- gsub('[^a-z0-9]', '_', tolower(url))
cat(fname)

Kurztest

Rückblick: Paginierung und höfliches Scraping

# Multi-page scraping template:
# urls <- paste0(base, 1:n_pages)
# results <- purrr::map(urls, function(url) {
#   Sys.sleep(runif(1, 1, 2))  # polite delay
#   page <- read_html(url)
#   extract_data(page)          # your parser
# })
# df <- dplyr::bind_rows(results)

# Key principles:
# 1. Respect robots.txt
# 2. Add delays (Sys.sleep)
# 3. Set honest User-Agent
# 4. Cache HTML during development
# 5. Handle errors with tryCatch
cat('Polite scraping = sustainable scraping')