Работа с REST API в R

Концепции REST API

library(httr2)

# REST API anatomy:
# Base URL:  https://api.example.com/v1
# Resources: /users, /products, /orders
# Methods:
#   GET    /users       -> list users
#   POST   /users       -> create user
#   GET    /users/42    -> get user 42
#   PUT    /users/42    -> update user 42
#   DELETE /users/42    -> delete user 42

# Query parameters for filtering/pagination:
# GET /users?page=2&size=20&sort=name

cat('REST = stateless + resource-based + HTTP methods')

Аутентификация с токеном Bearer

library(httr2)

# Store token securely in .Renviron:
# GITHUB_PAT=ghp_your_token_here

# Access at runtime:
token <- Sys.getenv('GITHUB_PAT')
if (nchar(token) == 0) token <- 'demo_token'

# Use in requests:
# resp <- request('https://api.github.com/user') |>
#   req_auth_bearer_token(token) |>
#   req_headers('Accept' = 'application/vnd.github.v3+json') |>
#   req_perform() |>
#   resp_check_status()

# result <- resp_body_json(resp)
# result$login  # your GitHub username
cat('Token from env:', if(nchar(token)>0) 'found' else 'missing')

Создание повторно используемого клиента API

library(httr2)

# API client constructor
new_api_client <- function(base_url, token) {
  list(
    base_req = request(base_url) |>
      req_auth_bearer_token(token) |>
      req_headers('Accept' = 'application/json') |>
      req_retry(max_tries = 3)
  )
}

# Method: GET /users
get_users <- function(client, page = 1, size = 20) {
  resp <- client$base_req |>
    req_url_path_append('users') |>
    req_url_query(page = page, size = size) |>
    req_perform() |>
    resp_check_status()
  resp_body_json(resp, simplifyVector = TRUE)
}

# client <- new_api_client('https://api.example.com', token)
# users <- get_users(client, page = 1)
cat('Reusable client pattern: base request + methods')

Разбиение на страницы с resp_link_url()

library(httr2)

# Generic paginator using Link headers
fetch_all_pages <- function(initial_url, token, max_pages = 50) {
  all_results <- list()
  next_url <- initial_url
  page <- 1
  while (!is.null(next_url) && page <= max_pages) {
    resp <- request(next_url) |>
      req_auth_bearer_token(token) |>
      req_perform() |>
      resp_check_status()
    all_results[[page]] <- resp_body_json(resp, simplifyVector = TRUE)
    # Follow Link: <url>; rel='next' header
    next_url <- tryCatch(
      resp_link_url(resp, 'next'),
      error = function(e) NULL
    )
    page <- page + 1
  }
  do.call(rbind, all_results)
}
cat('resp_link_url() follows RFC 5988 pagination')

Разбиение на страницы по курсору

library(httr2)

# Cursor-based pagination pattern
fetch_cursor_pages <- function(base_url, token, max_pages = 100) {
  all_data <- list()
  cursor <- NULL
  page <- 1
  repeat {
    req <- request(base_url) |>
      req_auth_bearer_token(token)
    if (!is.null(cursor))
      req <- req |> req_url_query(cursor = cursor)
    resp <- req |> req_perform() |> resp_check_status()
    body <- resp_body_json(resp)
    all_data[[page]] <- body$results
    cursor <- body$next_cursor  # NULL if last page
    if (is.null(cursor) || page >= max_pages) break
    page <- page + 1
  }
  do.call(c, all_data)
}
cat('Cursor pagination: safer for large/changing datasets')

Корректная обработка ошибок

library(httr2)

safe_api_call <- function(req) {
  tryCatch(
    req |> req_perform() |> resp_check_status(),
    httr2_http_401 = function(e) {
      stop('Authentication failed. Check your token.')
    },
    httr2_http_403 = function(e) {
      stop('Forbidden. Insufficient permissions.')
    },
    httr2_http_404 = function(e) {
      message('Resource not found, returning NULL')
      return(NULL)
    },
    httr2_http_429 = function(e) {
      stop('Rate limit exceeded. Try again later.')
    },
    error = function(e) {
      stop(paste('Request failed:', conditionMessage(e)))
    }
  )
}
cat('Match on specific httr2_http_NNN conditions')

Кэширование ответов API

library(httr2)

# httr2 built-in disk cache
# resp <- request('https://api.example.com/static-data') |>
#   req_cache(tempdir(), max_age = 3600) |>  # 1 hour TTL
#   req_perform()

# Manual cache pattern
cached_api_call <- function(url, cache_file, max_age = 3600) {
  if (file.exists(cache_file)) {
    age <- as.numeric(Sys.time() - file.mtime(cache_file))
    if (age < max_age) {
      cat('Cache hit\n')
      return(readRDS(cache_file))
    }
  }
  cat('Cache miss, fetching...\n')
  # result <- resp_body_json(request(url) |> req_perform())
  # saveRDS(result, cache_file)
  # result
}
cached_api_call('https://example.com/api', 'cache.rds')

Параллельные запросы с req_perform_parallel()

library(httr2)

# Build a list of requests
item_ids <- 1:5
reqs <- lapply(item_ids, function(id) {
  request(paste0('https://jsonplaceholder.typicode.com/todos/', id))
})

# Execute all in parallel (requires internet)
# resps <- req_perform_parallel(
#   reqs,
#   on_error = 'continue',  # skip failures
#   progress = TRUE
# )
# results <- lapply(resps, resp_body_json)
# titles <- sapply(results, function(r) r$title)
# print(titles)

# on_error options:
# 'stop'     -> abort on first failure
# 'continue' -> collect errors, keep going
cat('Parallel: much faster for many independent calls')

Работа с ограничениями частоты запросов

library(httr2)

respect_rate_limit <- function(resp) {
  # Check remaining calls
  remaining <- resp_header(resp, 'x-ratelimit-remaining')
  if (!is.na(remaining) && as.integer(remaining) < 5) {
    reset_at <- as.integer(
      resp_header(resp, 'x-ratelimit-reset')
    )
    wait_secs <- max(0, reset_at - as.integer(Sys.time()))
    cat(sprintf('Rate limit nearly exhausted. Waiting %ds\n', wait_secs))
    # Sys.sleep(wait_secs)
  }
  # Handle 429 Retry-After header
  if (resp_status(resp) == 429) {
    retry_after <- resp_header(resp, 'retry-after')
    cat(sprintf('Rate limited. Retry after %s seconds\n', retry_after))
  }
  resp
}
cat('Always respect X-RateLimit-* headers')

Полный клиент API: пример с GitHub

library(httr2)

# GitHub API client
github_repos <- function(username, token = NULL, n_pages = 3) {
  base <- 'https://api.github.com'
  req_base <- request(base) |>
    req_headers(
      'Accept'     = 'application/vnd.github.v3+json',
      'User-Agent' = 'R-API-Client/1.0'
    )
  if (!is.null(token))
    req_base <- req_base |> req_auth_bearer_token(token)

  all_repos <- list()
  for (page in seq_len(n_pages)) {
    # resp <- req_base |>
    #   req_url_path_append('users', username, 'repos') |>
    #   req_url_query(page = page, per_page = 30, sort = 'updated') |>
    #   req_perform() |> resp_check_status()
    # repos <- resp_body_json(resp, simplifyVector = TRUE)
    # if (length(repos) == 0) break
    # all_repos[[page]] <- repos
    cat(sprintf('Would fetch page %d for %s\n', page, username))
  }
  do.call(rbind, all_repos)
}
github_repos('hadley')

Учётные данные клиента OAuth 2.0

library(httr2)

# OAuth 2.0 Client Credentials flow:
# client <- oauth_client(
#   id     = Sys.getenv('CLIENT_ID'),
#   secret = Sys.getenv('CLIENT_SECRET'),
#   token_url = 'https://auth.example.com/oauth/token'
# )

# Automatic token management:
# resp <- request('https://api.example.com/data') |>
#   req_oauth_client_credentials(client) |>
#   req_perform()
# httr2 automatically:
# 1. Gets access token using client credentials
# 2. Adds 'Authorization: Bearer <token>' header
# 3. Refreshes token when expired

cat('req_oauth_client_credentials() handles token lifecycle')
cat('Token is cached in memory automatically')

Быстрая проверка

Итоги: использование REST API

library(httr2)

# REST API client template:
make_api_call <- function(url, token,
                          method = 'GET',
                          body = NULL,
                          query = list()) {
  req <- request(url) |>
    req_method(method) |>
    req_auth_bearer_token(token) |>
    req_headers('Accept' = 'application/json') |>
    req_retry(max_tries = 3) |>
    req_throttle(rate = 10/60)
  if (length(query) > 0)
    req <- do.call(req_url_query, c(list(req), query))
  if (!is.null(body))
    req <- req |> req_body_json(body)
  req |> req_perform() |> resp_check_status()
}
cat('Template: auth + retry + throttle + error check')