diff --git a/tests/benchmarks/test_urls.py b/tests/benchmarks/test_urls.py new file mode 100644 index 000000000..acc1d4d80 --- /dev/null +++ b/tests/benchmarks/test_urls.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from html import escape +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request +from scrapy.http import HtmlResponse +from scrapy.linkextractors import LinkExtractor +from scrapy.utils.request import fingerprint + +if TYPE_CHECKING: + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +RESPONSE_URL = "https://www.example.com/catalogue/page-1.html" + +# Links that each scenario returns for the benchmark page. They are fewer than +# the anchors of the page because links to images, to other non-crawlable files +# and to non-HTTP schemes are rejected, and, except in the scenario that keeps +# duplicates, because the links that the navigation repeats are collapsed. +LINKS = 63 +DUPLICATE_LINKS = 88 +CANONICAL_LINKS = 60 +FILTERED_LINKS = 45 + +# Requests built from LINKS links that point to a different resource. +# Canonicalization maps the rest to one that another link already covers, e.g. +# two fragments of a page, or two spellings of one percent-escape. +FINGERPRINTS = 60 + + +def _read_corpus() -> tuple[list[str], list[str]]: + """Return the URLs of ``urls.txt``, and its first group of URLs. + + The first group is the site navigation, which the benchmark page repeats. + """ + groups: list[list[str]] = [[]] + for line in (Path(__file__).parent / "urls.txt").read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + if groups[-1]: + groups.append([]) + continue + groups[-1].append(line) + urls = [url for group in groups for url in group] + return urls, groups[0] + + +def _build_page(urls: list[str], navigation: list[str]) -> bytes: + """Return an HTML page that links to *urls*. + + Every link is surrounded by the markup of a product listing, so that + benchmarks also cover walking over the elements and attributes that a real + page puts between links. + """ + + def item(index: int, url: str) -> str: + href = escape(url) + return ( + f'
  • ' + f'Product {index}' + f'

    Product {index}

    ' + f'

    A description of product {index}.

    ' + f"
    • " + ) + + def nav(urls: list[str]) -> str: + links = "".join(f'{escape(url)}' for url in urls) + return f'' + + items = "".join(item(index, url) for index, url in enumerate(urls)) + return ( + "Catalogue" + f'' + f'{nav(navigation)}{nav(navigation)}' + "" + ).encode() + + +URLS, NAVIGATION = _read_corpus() +BODY = _build_page(URLS, NAVIGATION) + + +def _response() -> HtmlResponse: + return HtmlResponse(RESPONSE_URL, body=BODY, encoding="utf-8") + + +@pytest.mark.parametrize( + ("kwargs", "links"), + [ + pytest.param({}, LINKS, id="default"), + pytest.param({"unique": False}, DUPLICATE_LINKS, id="duplicates"), + pytest.param({"canonicalize": True}, CANONICAL_LINKS, id="canonicalize"), + pytest.param( + { + "allow": r"/catalogue/", + "deny": r"/legal/", + "allow_domains": ["example.com", "www.example.com"], + }, + FILTERED_LINKS, + id="filtered", + ), + ], +) +def test_extract_links( + benchmark: BenchmarkFixture, kwargs: dict[str, Any], links: int +) -> None: + """Extraction of every link of a page. + + The scenarios cover the choices that change which work dominates: + deduplication and canonicalization both build a key for every link, and the + filters of a configured extractor reject links before the later checks, + which the default extractor reaches for every link. + """ + link_extractor = LinkExtractor(**kwargs) + + def run() -> None: + assert len(link_extractor.extract_links(_response())) == links + + benchmark(run) + + +EXTRACTED_URLS = [link.url for link in LinkExtractor().extract_links(_response())] + + +def test_requests(benchmark: BenchmarkFixture) -> None: + """Building a request for every link of a page.""" + + def run() -> None: + assert len([Request(url) for url in EXTRACTED_URLS]) == LINKS + + benchmark(run) + + +def test_fingerprints(benchmark: BenchmarkFixture) -> None: + """Fingerprinting the request of every link of a page. + + Requests are built here as well, and not once for all rounds, because + fingerprints are cached per request object. + """ + + def run() -> None: + assert ( + len({fingerprint(Request(url)) for url in EXTRACTED_URLS}) == FINGERPRINTS + ) + + benchmark(run) diff --git a/tests/benchmarks/urls.txt b/tests/benchmarks/urls.txt new file mode 100644 index 000000000..6ef6939f1 --- /dev/null +++ b/tests/benchmarks/urls.txt @@ -0,0 +1,130 @@ +# Link targets for the URL benchmarks, as they would appear in the href +# attribute of a page at https://www.example.com/catalogue/page-1.html. +# +# Cost per URL varies by shape: the number of query parameters drives the +# parsing and re-encoding of the query string, non-ASCII characters and +# unescaped characters drive percent-encoding, and non-default ports, dot +# segments and uppercase host names drive normalization. A corpus of uniform +# URLs would therefore measure one shape and miss the others, so this one +# covers each of them, in roughly the proportion of a real listing page. +# +# Blank lines and lines starting with "#" are ignored. + +# Site navigation. These also appear in a second copy of the navigation at the +# end of the page, so that deduplication has duplicates to collapse. +/ +/index.html +/about-us +/contact +/catalogue/ +/catalogue/page-2.html +/catalogue/page-3.html +/help/faq +/help/shipping-and-returns +/legal/terms +/legal/privacy + +# Relative paths of increasing depth. +detail.html +./detail.html +../catalogue/page-4.html +../../index.html +/catalogue/category/books/fiction/index.html +/catalogue/category/books/travel/mystery/historical/index.html +/a/b/c/d/e/f/g/h/i/j/k/index.html + +# One query parameter. +/catalogue/search?q=book +/catalogue/page-1.html?page=2 +/catalogue/detail?id=1042 + +# Several query parameters, in an order that canonicalization changes. +/catalogue/search?q=book&sort=price +/catalogue/search?sort=price&q=book +/catalogue/search?q=book&sort=price&page=3&per_page=20&in_stock=1 +/catalogue/search?zone=eu&q=book&min=10&max=90&sort=rating&page=2&view=grid&lang=en¤cy=EUR&ref=nav + +# Repeated keys, blank values and a bare key. +/catalogue/search?tag=fiction&tag=travel&tag=history +/catalogue/search?q=&sort= +/catalogue/search?featured + +# Characters that need percent-encoding. +/catalogue/search?q=cheap books +/catalogue/detail/a book about books.html +/catalogue/search?q=100%+cotton +/catalogue/search?price=%3E10&title=A%20%26%20B + +# Percent-escapes that are already valid, in both cases. +/catalogue/detail/%C3%A9dition-limit%C3%A9e.html +/catalogue/detail/%c3%a9dition-limit%c3%a9e.html +/catalogue/detail/%7Especial.html + +# Non-ASCII in the path and in the query. +/catalogue/detail/édition-limitée.html +/catalogue/search?q=édition +/catalogue/búsqueda?q=libro&categoría=ficción +/カタログ/詳細.html + +# Internationalized host names, encoded and decoded. +https://例え.テスト/catalogue/page-1.html +https://xn--r8jz45g.xn--zckzah/catalogue/page-2.html + +# Absolute URLs on the same host, on other hosts, and protocol-relative. +https://www.example.com/catalogue/page-5.html +https://www.example.com/catalogue/detail?id=1043 +http://www.example.com/catalogue/page-6.html +https://shop.example.com/catalogue/page-1.html +https://www.example.org/reviews/1042 +https://books.toscrape.com/catalogue/page-1.html +//cdn.example.com/catalogue/page-7.html +//www.example.com/catalogue/page-8.html + +# Ports, including the default one for the scheme. +https://www.example.com:443/catalogue/page-9.html +http://www.example.com:80/catalogue/page-10.html +https://staging.example.com:8443/catalogue/page-1.html + +# Host name case, which normalization lowercases. +https://WWW.EXAMPLE.COM/catalogue/Page-11.html +HTTPS://www.example.com/catalogue/page-12.html + +# Dot segments, empty segments and trailing slashes, which WHATWG +# normalization resolves and the standard library keeps. +/catalogue/../catalogue/page-13.html +/catalogue/./page-14.html +/catalogue//page-15.html +/catalogue/category/ +/catalogue/category + +# Fragments, which canonicalization drops and the deduplication key keeps. +/catalogue/page-16.html#reviews +/catalogue/page-16.html#description +/catalogue/page-17.html# +#top + +# Path parameters, where the semicolon is not the last segment. +/catalogue;sessionid=abc123/page-18.html +/catalogue/page-19.html;sessionid=abc123 + +# User information in the authority. +https://user:password@files.example.com/catalogue/page-1.html + +# A long URL, of the length that tracking parameters reach. +/catalogue/search?q=book&utm_source=newsletter&utm_medium=email&utm_campaign=spring-sale-2026&utm_term=fiction%20paperback&utm_content=hero-banner-variant-b&session=6f1c9a2e4b7d8f0a1c3e5d7b9f2a4c6e&ref=https%3A%2F%2Fwww.example.org%2Freviews%2F1042&page=2&sort=relevance + +# Extensions that the default deny_extensions rejects, and one compound +# extension, which only matches as a whole. +/media/cover-1042.jpg +/media/cover-1042.PNG +/media/catalogue.pdf +/static/style.css +/static/app.js +/downloads/catalogue.tar.gz +/downloads/catalogue.zip + +# Schemes that are not crawlable, which are rejected before any parsing. +mailto:orders@example.com +javascript:void(0) +tel:+441234567890 +data:text/plain,hello