Web Scraping with Scrapy: Complete Python Guide

Web Scraping with Scrapy: Complete Python Guide

Scrapy is the most powerful web scraping framework for Python. Unlike scripts using Requests + BeautifulSoup, Scrapy offers: async architecture (Twisted), middlewares, data pipelines, automatic robots.txt handling, throttling, and multi-format export. Ideal for serious data extraction projects.

Installation and Project Setup

pip install scrapy

# Create new project
scrapy startproject my_scraper
cd my_scraper

# Generated structure:
# my_scraper/
#   scrapy.cfg
#   my_scraper/
#     __init__.py
#     items.py        ← data definitions
#     middlewares.py  ← middleware logic
#     pipelines.py    ← item processing
#     settings.py     ← global configuration
#     spiders/        ← spiders go here

Your First Spider

# my_scraper/spiders/books_spider.py
import scrapy

class BooksSpider(scrapy.Spider):
    name       = "books"
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):
        for book in response.css("article.product_pod"):
            yield {
                "title":       book.css("h3 a::attr(title)").get(),
                "price":       book.css("p.price_color::text").get(),
                "availability": book.css("p.availability::text").get().strip(),
                "rating":      book.css("p.star-rating::attr(class)").get().split()[-1],
                "detail_url":  response.urljoin(book.css("h3 a::attr(href)").get()),
            }

        # Follow to next page (pagination)
        next_page = response.css("li.next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

Run the spider:

# Print to screen
scrapy crawl books

# Export to JSON
scrapy crawl books -o books.json

# Export to CSV
scrapy crawl books -o books.csv

# JSONLines (one JSON per line — better for large files)
scrapy crawl books -o books.jsonl

CSS and XPath Selectors

def parse(self, response):
    # ── CSS Selectors ────────────────────────────────────
    # Element text
    title = response.css("h1.product-title::text").get()

    # Attribute
    img_url = response.css("img.cover::attr(src)").get()

    # All elements (list)
    all_prices = response.css("span.price::text").getall()

    # Cleaned text
    desc = response.css("div.description p::text").get("").strip()

    # ── XPath Selectors ──────────────────────────────────
    # More powerful for complex structures
    author = response.xpath("//span[@itemprop='author']/text()").get()

    # Text containing a string
    notice = response.xpath("//*[contains(text(), 'Sale')]//text()").getall()

    # Navigate to parent
    price_parent = response.xpath("//span[@class='price']/parent::div/text()").get()

    # Mix CSS and XPath in the same spider
    for product in response.css("div.product"):
        yield {
            "name":     product.css("h2::text").get(),
            "price":    product.xpath(".//span[@class='price']/text()").get(),
            "in_stock": "yes" if product.css(".in-stock") else "no",
        }

Items: Structuring Extracted Data

# my_scraper/items.py
import scrapy

class BookItem(scrapy.Item):
    title       = scrapy.Field()
    author      = scrapy.Field()
    price       = scrapy.Field(serializer=float)
    isbn        = scrapy.Field()
    genre       = scrapy.Field()
    rating      = scrapy.Field(serializer=int)
    url         = scrapy.Field()
    scraped_date = scrapy.Field()

# In the spider — use Items instead of plain dicts
from my_scraper.items import BookItem
from datetime import date

class BooksSpider(scrapy.Spider):
    name = "books_items"
    start_urls = ["https://books.toscrape.com"]

    RATINGS = {"One": 1, "Two": 2, "Three": 3, "Four": 4, "Five": 5}

    def parse(self, response):
        for book in response.css("article.product_pod"):
            item                = BookItem()
            item['title']       = book.css("h3 a::attr(title)").get()
            item['price']       = book.css("p.price_color::text").get("").replace("£", "")
            item['rating']      = self.RATINGS.get(
                book.css("p.star-rating::attr(class)").get("").split()[-1], 0
            )
            item['url']         = response.urljoin(book.css("h3 a::attr(href)").get())
            item['scraped_date'] = str(date.today())
            yield item

Multi-Page Spider with Detail Pages

import scrapy
from my_scraper.items import BookItem

class BooksDetailSpider(scrapy.Spider):
    name = "books_detail"
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):
        """Listing page — follow each book link."""
        for book in response.css("article.product_pod"):
            url = response.urljoin(book.css("h3 a::attr(href)").get())
            yield response.follow(url, self.parse_book)

        next_page = response.css("li.next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

    def parse_book(self, response):
        """Detail page — extract full information."""
        item          = BookItem()
        item['title'] = response.css("h1::text").get()
        item['price'] = response.css("p.price_color::text").get("").replace("£", "")
        item['url']   = response.url

        table = {}
        for row in response.css("table.table-striped tr"):
            key   = row.css("th::text").get("")
            value = row.css("td::text").get("")
            table[key] = value

        item['isbn']  = table.get("UPC", "")
        item['genre'] = response.css("ul.breadcrumb li:nth-child(3) a::text").get()
        yield item

Data Pipelines

# my_scraper/pipelines.py
import re
import sqlite3
from itemadapter import ItemAdapter

class CleaningPipeline:
    """Clean and normalize scraped data."""
    def process_item(self, item, spider):
        adapter = ItemAdapter(item)

        if adapter.get('price'):
            price_str   = adapter['price']
            clean_price = re.sub(r'[^\d.,]', '', price_str).replace(',', '.')
            adapter['price'] = float(clean_price) if clean_price else 0.0

        if adapter.get('title'):
            adapter['title'] = adapter['title'].strip()

        return item

class SQLitePipeline:
    """Save items to SQLite database."""
    def open_spider(self, spider):
        self.conn   = sqlite3.connect('books.db')
        self.cursor = self.conn.cursor()
        self.cursor.execute("""
            CREATE TABLE IF NOT EXISTS books (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                title TEXT, author TEXT, price REAL,
                isbn TEXT UNIQUE, genre TEXT,
                rating INTEGER, url TEXT, scraped_date TEXT
            )
        """)
        self.conn.commit()

    def close_spider(self, spider):
        self.conn.close()

    def process_item(self, item, spider):
        adapter = ItemAdapter(item)
        try:
            self.cursor.execute("""
                INSERT OR REPLACE INTO books
                (title, author, price, isbn, genre, rating, url, scraped_date)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                adapter.get('title'),  adapter.get('author'),
                adapter.get('price'),  adapter.get('isbn'),
                adapter.get('genre'),  adapter.get('rating'),
                adapter.get('url'),    adapter.get('scraped_date'),
            ))
            self.conn.commit()
        except Exception as e:
            spider.logger.error(f"SQLite error: {e}")
        return item

Activate in settings.py:

ITEM_PIPELINES = {
    'my_scraper.pipelines.CleaningPipeline': 100,
    'my_scraper.pipelines.SQLitePipeline':   300,
}

Recommended Configuration (settings.py)

BOT_NAME = 'my_scraper'
ROBOTSTXT_OBEY = True

# AutoThrottle (be respectful)
AUTOTHROTTLE_ENABLED             = True
AUTOTHROTTLE_START_DELAY         = 1.0
AUTOTHROTTLE_MAX_DELAY           = 10.0
AUTOTHROTTLE_TARGET_CONCURRENCY  = 2.0

# Default headers
DEFAULT_REQUEST_HEADERS = {
    'Accept':          'text/html,application/xhtml+xml',
    'Accept-Language': 'en-US,en;q=0.9',
    'User-Agent':      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}

# Response caching (avoid re-downloading during development)
HTTPCACHE_ENABLED          = True
HTTPCACHE_EXPIRATION_SECS  = 3600
HTTPCACHE_DIR              = '.scrapy_cache'

# Concurrency
CONCURRENT_REQUESTS             = 8
CONCURRENT_REQUESTS_PER_DOMAIN  = 4
DOWNLOAD_DELAY                  = 0.5

LOG_LEVEL = 'INFO'
LOG_FILE  = 'scrapy.log'

Programmatic Spider (No Project Structure)

import scrapy
from scrapy.crawler import CrawlerProcess

class QuickSpider(scrapy.Spider):
    name = "quick"
    start_urls = ["https://example.com"]

    custom_settings = {
        'FEEDS': {'result.json': {'format': 'json'}},
        'ROBOTSTXT_OBEY': True,
        'LOG_LEVEL': 'WARNING',
    }

    def parse(self, response):
        yield {
            'title': response.css("title::text").get(),
            'url':   response.url,
            'links': response.css("a::attr(href)").getall()[:10],
        }

process = CrawlerProcess()
process.crawl(QuickSpider)
process.start()

Additional Resource

For converting scraped data between CSV, JSON, Excel, and other formats without additional coding, use KaijuConverter — free and no registration required.

Related conversions

Frequent conversions across the catalogue:

• JPG to PNG
• MP4 to MP3
• PDF to DOCX
• PNG to JPG
• MP3 to WAV
• DOCX to PDF