Scrapy 스파이더 구축

Scrapy란 무엇입니까

# pip install scrapy

# Create a project:
# scrapy startproject myspider

# myspider/
#   scrapy.cfg
#   myspider/
#     settings.py
#     spiders/
#       quotes_spider.py

Spider 만들기

import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ["https://quotes.toscrape.com"]

    def parse(self, response):
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
            }

CSS 및 XPath 선택자

def parse(self, response):
    # CSS:
    titles = response.css("h2.title::text").getall()

    # XPath:
    links = response.xpath("//a/@href").getall()

    yield {"titles": titles, "links": links}

링크 따라가기

import scrapy

class Spider(scrapy.Spider):
    name = "crawler"
    start_urls = ["https://quotes.toscrape.com"]

    def parse(self, response):
        for quote in response.css("div.quote"):
            yield {"text": quote.css("span.text::text").get()}

        next_page = response.css("li.next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

항목과 ItemLoaders

import scrapy

class ProductItem(scrapy.Item):
    name  = scrapy.Field()
    price = scrapy.Field()
    url   = scrapy.Field()

def parse(self, response):
    item = ProductItem()
    item["name"]  = response.css("h1::text").get(strip=True)
    item["price"] = response.css(".price::text").re_first(r"[\d.]+")
    item["url"]   = response.url
    yield item

Spider 실행하기

# Run and print to console:
# scrapy crawl quotes

# Save to JSON:
# scrapy crawl quotes -o quotes.json

# Save to CSV:
# scrapy crawl quotes -o quotes.csv

# Scrapy shell for interactive testing:
# scrapy shell "https://quotes.toscrape.com"

설정: 다운로드 지연과 동시 요청 수

# settings.py
DOWNLOAD_DELAY = 1          # 1 second between requests
CONCURRENT_REQUESTS = 8     # max 8 at a time
CONCURRENT_REQUESTS_PER_DOMAIN = 2
ROBOTSTXT_OBEY = True       # respect robots.txt

미들웨어

# settings.py
DOWNLOADER_MIDDLEWARES = {
    "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
    "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
    "myspider.middlewares.RotateUserAgentMiddleware": 400,
}

항목 파이프라인

# pipelines.py
class CleanPipeline:
    def process_item(self, item, spider):
        item["name"] = item["name"].strip()
        return item

class DBPipeline:
    def open_spider(self, spider):
        self.conn = connect_db()

    def process_item(self, item, spider):
        self.conn.insert(dict(item))
        return item

로그인 처리

import scrapy

class AuthSpider(scrapy.Spider):
    name = "auth"
    login_url = "https://example.com/login"

    def start_requests(self):
        yield scrapy.Request(self.login_url, self.login)

    def login(self, response):
        yield scrapy.FormRequest.from_response(
            response,
            formdata={"user": "me", "pass": "secret"},
            callback=self.parse
        )

JS 페이지용 scrapy-playwright

# pip install scrapy-playwright
# playwright install

# settings.py:
# DOWNLOAD_HANDLERS = {
#     "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
#     "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
# }

# In spider:
# yield scrapy.Request(url, meta={"playwright": True})

빠른 확인

복습