mirror of https://github.com/scrapy/scrapy.git
Add URL benchmarks (#7914)
This commit is contained in:
Adrian
GitHub
parent
4a69e48f0f
commit
fc9c505e79
|
|
@ -0,0 +1,152 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from html import escape
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
|
||||
|
||||
pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
|
||||
|
||||
RESPONSE_URL = "https://www.example.com/catalogue/page-1.html"
|
||||
|
||||
# Links that each scenario returns for the benchmark page. They are fewer than
|
||||
# the anchors of the page because links to images, to other non-crawlable files
|
||||
# and to non-HTTP schemes are rejected, and, except in the scenario that keeps
|
||||
# duplicates, because the links that the navigation repeats are collapsed.
|
||||
LINKS = 63
|
||||
DUPLICATE_LINKS = 88
|
||||
CANONICAL_LINKS = 60
|
||||
FILTERED_LINKS = 45
|
||||
|
||||
# Requests built from LINKS links that point to a different resource.
|
||||
# Canonicalization maps the rest to one that another link already covers, e.g.
|
||||
# two fragments of a page, or two spellings of one percent-escape.
|
||||
FINGERPRINTS = 60
|
||||
|
||||
|
||||
def _read_corpus() -> tuple[list[str], list[str]]:
|
||||
"""Return the URLs of ``urls.txt``, and its first group of URLs.
|
||||
|
||||
The first group is the site navigation, which the benchmark page repeats.
|
||||
"""
|
||||
groups: list[list[str]] = [[]]
|
||||
for line in (Path(__file__).parent / "urls.txt").read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
if groups[-1]:
|
||||
groups.append([])
|
||||
continue
|
||||
groups[-1].append(line)
|
||||
urls = [url for group in groups for url in group]
|
||||
return urls, groups[0]
|
||||
|
||||
|
||||
def _build_page(urls: list[str], navigation: list[str]) -> bytes:
|
||||
"""Return an HTML page that links to *urls*.
|
||||
|
||||
Every link is surrounded by the markup of a product listing, so that
|
||||
benchmarks also cover walking over the elements and attributes that a real
|
||||
page puts between links.
|
||||
"""
|
||||
|
||||
def item(index: int, url: str) -> str:
|
||||
href = escape(url)
|
||||
return (
|
||||
f'<li class="product" data-index="{index}">'
|
||||
f'<img src="/media/thumbnail-{index}.jpg" alt="Product {index}" '
|
||||
f'width="128" height="128">'
|
||||
f'<h3><a href="{href}">Product {index}</a></h3>'
|
||||
f'<p class="description">A description of product {index}.</p>'
|
||||
f"</li>"
|
||||
)
|
||||
|
||||
def nav(urls: list[str]) -> str:
|
||||
links = "".join(f'<a href="{escape(url)}">{escape(url)}</a>' for url in urls)
|
||||
return f'<nav class="site">{links}</nav>'
|
||||
|
||||
items = "".join(item(index, url) for index, url in enumerate(urls))
|
||||
return (
|
||||
"<!DOCTYPE html><html><head><title>Catalogue</title>"
|
||||
f'<base href="{RESPONSE_URL}"></head><body>'
|
||||
f'{nav(navigation)}<ul class="products">{items}</ul>{nav(navigation)}'
|
||||
"</body></html>"
|
||||
).encode()
|
||||
|
||||
|
||||
URLS, NAVIGATION = _read_corpus()
|
||||
BODY = _build_page(URLS, NAVIGATION)
|
||||
|
||||
|
||||
def _response() -> HtmlResponse:
|
||||
return HtmlResponse(RESPONSE_URL, body=BODY, encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "links"),
|
||||
[
|
||||
pytest.param({}, LINKS, id="default"),
|
||||
pytest.param({"unique": False}, DUPLICATE_LINKS, id="duplicates"),
|
||||
pytest.param({"canonicalize": True}, CANONICAL_LINKS, id="canonicalize"),
|
||||
pytest.param(
|
||||
{
|
||||
"allow": r"/catalogue/",
|
||||
"deny": r"/legal/",
|
||||
"allow_domains": ["example.com", "www.example.com"],
|
||||
},
|
||||
FILTERED_LINKS,
|
||||
id="filtered",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_links(
|
||||
benchmark: BenchmarkFixture, kwargs: dict[str, Any], links: int
|
||||
) -> None:
|
||||
"""Extraction of every link of a page.
|
||||
|
||||
The scenarios cover the choices that change which work dominates:
|
||||
deduplication and canonicalization both build a key for every link, and the
|
||||
filters of a configured extractor reject links before the later checks,
|
||||
which the default extractor reaches for every link.
|
||||
"""
|
||||
link_extractor = LinkExtractor(**kwargs)
|
||||
|
||||
def run() -> None:
|
||||
assert len(link_extractor.extract_links(_response())) == links
|
||||
|
||||
benchmark(run)
|
||||
|
||||
|
||||
EXTRACTED_URLS = [link.url for link in LinkExtractor().extract_links(_response())]
|
||||
|
||||
|
||||
def test_requests(benchmark: BenchmarkFixture) -> None:
|
||||
"""Building a request for every link of a page."""
|
||||
|
||||
def run() -> None:
|
||||
assert len([Request(url) for url in EXTRACTED_URLS]) == LINKS
|
||||
|
||||
benchmark(run)
|
||||
|
||||
|
||||
def test_fingerprints(benchmark: BenchmarkFixture) -> None:
|
||||
"""Fingerprinting the request of every link of a page.
|
||||
|
||||
Requests are built here as well, and not once for all rounds, because
|
||||
fingerprints are cached per request object.
|
||||
"""
|
||||
|
||||
def run() -> None:
|
||||
assert (
|
||||
len({fingerprint(Request(url)) for url in EXTRACTED_URLS}) == FINGERPRINTS
|
||||
)
|
||||
|
||||
benchmark(run)
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
# Link targets for the URL benchmarks, as they would appear in the href
|
||||
# attribute of a page at https://www.example.com/catalogue/page-1.html.
|
||||
#
|
||||
# Cost per URL varies by shape: the number of query parameters drives the
|
||||
# parsing and re-encoding of the query string, non-ASCII characters and
|
||||
# unescaped characters drive percent-encoding, and non-default ports, dot
|
||||
# segments and uppercase host names drive normalization. A corpus of uniform
|
||||
# URLs would therefore measure one shape and miss the others, so this one
|
||||
# covers each of them, in roughly the proportion of a real listing page.
|
||||
#
|
||||
# Blank lines and lines starting with "#" are ignored.
|
||||
|
||||
# Site navigation. These also appear in a second copy of the navigation at the
|
||||
# end of the page, so that deduplication has duplicates to collapse.
|
||||
/
|
||||
/index.html
|
||||
/about-us
|
||||
/contact
|
||||
/catalogue/
|
||||
/catalogue/page-2.html
|
||||
/catalogue/page-3.html
|
||||
/help/faq
|
||||
/help/shipping-and-returns
|
||||
/legal/terms
|
||||
/legal/privacy
|
||||
|
||||
# Relative paths of increasing depth.
|
||||
detail.html
|
||||
./detail.html
|
||||
../catalogue/page-4.html
|
||||
../../index.html
|
||||
/catalogue/category/books/fiction/index.html
|
||||
/catalogue/category/books/travel/mystery/historical/index.html
|
||||
/a/b/c/d/e/f/g/h/i/j/k/index.html
|
||||
|
||||
# One query parameter.
|
||||
/catalogue/search?q=book
|
||||
/catalogue/page-1.html?page=2
|
||||
/catalogue/detail?id=1042
|
||||
|
||||
# Several query parameters, in an order that canonicalization changes.
|
||||
/catalogue/search?q=book&sort=price
|
||||
/catalogue/search?sort=price&q=book
|
||||
/catalogue/search?q=book&sort=price&page=3&per_page=20&in_stock=1
|
||||
/catalogue/search?zone=eu&q=book&min=10&max=90&sort=rating&page=2&view=grid&lang=en¤cy=EUR&ref=nav
|
||||
|
||||
# Repeated keys, blank values and a bare key.
|
||||
/catalogue/search?tag=fiction&tag=travel&tag=history
|
||||
/catalogue/search?q=&sort=
|
||||
/catalogue/search?featured
|
||||
|
||||
# Characters that need percent-encoding.
|
||||
/catalogue/search?q=cheap books
|
||||
/catalogue/detail/a book about books.html
|
||||
/catalogue/search?q=100%+cotton
|
||||
/catalogue/search?price=%3E10&title=A%20%26%20B
|
||||
|
||||
# Percent-escapes that are already valid, in both cases.
|
||||
/catalogue/detail/%C3%A9dition-limit%C3%A9e.html
|
||||
/catalogue/detail/%c3%a9dition-limit%c3%a9e.html
|
||||
/catalogue/detail/%7Especial.html
|
||||
|
||||
# Non-ASCII in the path and in the query.
|
||||
/catalogue/detail/édition-limitée.html
|
||||
/catalogue/search?q=édition
|
||||
/catalogue/búsqueda?q=libro&categoría=ficción
|
||||
/カタログ/詳細.html
|
||||
|
||||
# Internationalized host names, encoded and decoded.
|
||||
https://例え.テスト/catalogue/page-1.html
|
||||
https://xn--r8jz45g.xn--zckzah/catalogue/page-2.html
|
||||
|
||||
# Absolute URLs on the same host, on other hosts, and protocol-relative.
|
||||
https://www.example.com/catalogue/page-5.html
|
||||
https://www.example.com/catalogue/detail?id=1043
|
||||
http://www.example.com/catalogue/page-6.html
|
||||
https://shop.example.com/catalogue/page-1.html
|
||||
https://www.example.org/reviews/1042
|
||||
https://books.toscrape.com/catalogue/page-1.html
|
||||
//cdn.example.com/catalogue/page-7.html
|
||||
//www.example.com/catalogue/page-8.html
|
||||
|
||||
# Ports, including the default one for the scheme.
|
||||
https://www.example.com:443/catalogue/page-9.html
|
||||
http://www.example.com:80/catalogue/page-10.html
|
||||
https://staging.example.com:8443/catalogue/page-1.html
|
||||
|
||||
# Host name case, which normalization lowercases.
|
||||
https://WWW.EXAMPLE.COM/catalogue/Page-11.html
|
||||
HTTPS://www.example.com/catalogue/page-12.html
|
||||
|
||||
# Dot segments, empty segments and trailing slashes, which WHATWG
|
||||
# normalization resolves and the standard library keeps.
|
||||
/catalogue/../catalogue/page-13.html
|
||||
/catalogue/./page-14.html
|
||||
/catalogue//page-15.html
|
||||
/catalogue/category/
|
||||
/catalogue/category
|
||||
|
||||
# Fragments, which canonicalization drops and the deduplication key keeps.
|
||||
/catalogue/page-16.html#reviews
|
||||
/catalogue/page-16.html#description
|
||||
/catalogue/page-17.html#
|
||||
#top
|
||||
|
||||
# Path parameters, where the semicolon is not the last segment.
|
||||
/catalogue;sessionid=abc123/page-18.html
|
||||
/catalogue/page-19.html;sessionid=abc123
|
||||
|
||||
# User information in the authority.
|
||||
https://user:password@files.example.com/catalogue/page-1.html
|
||||
|
||||
# A long URL, of the length that tracking parameters reach.
|
||||
/catalogue/search?q=book&utm_source=newsletter&utm_medium=email&utm_campaign=spring-sale-2026&utm_term=fiction%20paperback&utm_content=hero-banner-variant-b&session=6f1c9a2e4b7d8f0a1c3e5d7b9f2a4c6e&ref=https%3A%2F%2Fwww.example.org%2Freviews%2F1042&page=2&sort=relevance
|
||||
|
||||
# Extensions that the default deny_extensions rejects, and one compound
|
||||
# extension, which only matches as a whole.
|
||||
/media/cover-1042.jpg
|
||||
/media/cover-1042.PNG
|
||||
/media/catalogue.pdf
|
||||
/static/style.css
|
||||
/static/app.js
|
||||
/downloads/catalogue.tar.gz
|
||||
/downloads/catalogue.zip
|
||||
|
||||
# Schemes that are not crawlable, which are rejected before any parsing.
|
||||
mailto:orders@example.com
|
||||
javascript:void(0)
|
||||
tel:+441234567890
|
||||
data:text/plain,hello
|
||||
Loading…
Reference in New Issue