diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..6930c4c1c --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,45 @@ +--- +name: codspeed + +on: + push: + branches: + - master + pull_request: + paths: + - scrapy/** + - tests/benchmarks/** + - .github/workflows/codspeed.yml + - tox.ini + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + benchmark: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # OIDC authentication with CodSpeed + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: '3.14' + - name: Install dependencies + run: | + pip install --upgrade pip + pip install --upgrade tox + tox -n -e benchmark + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: tox -e benchmark diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 0409b3ef2..566b34e50 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -14,14 +14,18 @@ jobs: tests: runs-on: macos-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13"] env: - TOXENV: py include: + - python-version: '3.14' + env: + TOXENV: py + coverage: true - python-version: '3.14' env: TOXENV: no-reactor @@ -41,6 +45,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 15f25d9b8..ad2bcfce7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -14,7 +14,7 @@ jobs: tests: runs-on: ubuntu-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: @@ -34,12 +34,15 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor + coverage: true - python-version: "3.14" env: TOXENV: no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -49,12 +52,15 @@ jobs: - python-version: "3.10.19" env: TOXENV: min + coverage: true - python-version: "3.10.19" env: TOXENV: min-default-reactor + coverage: true - python-version: "3.10.19" env: TOXENV: min-no-reactor + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -62,16 +68,20 @@ jobs: - python-version: "3.10.19" env: TOXENV: min-extra-deps + coverage: true - python-version: "3.10.19" env: TOXENV: min-botocore + coverage: true - python-version: "3.14" env: TOXENV: extra-deps + coverage: true - python-version: "3.14" env: TOXENV: no-reactor-extra-deps + coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: @@ -79,6 +89,7 @@ jobs: - python-version: "3.14" env: TOXENV: botocore + coverage: true steps: - uses: actions/checkout@v6 @@ -104,6 +115,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f413782bc..c33f96b12 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -14,7 +14,7 @@ jobs: tests: runs-on: windows-latest env: - PYTEST_ADDOPTS: -n auto + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} strategy: fail-fast: false matrix: @@ -34,6 +34,7 @@ jobs: - python-version: "3.14" env: TOXENV: py + coverage: true - python-version: "3.14" env: TOXENV: default-reactor @@ -68,6 +69,7 @@ jobs: tox - name: Upload coverage report + if: ${{ matrix.coverage }} uses: codecov/codecov-action@v5 - name: Upload test results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311df7052..c27348c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.8 + rev: 0.8.9 hooks: - id: sphinx-scrapy diff --git a/README.rst b/README.rst index 6235cb20c..651294add 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ :alt: Scrapy :width: 480px -|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| +|version| |python_version| |tests| |coverage| |conda| |deepwiki| .. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -15,17 +15,9 @@ :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu - :alt: Ubuntu - -.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS - -.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows - :alt: Windows +.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests + :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster + :alt: Tests .. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master diff --git a/conftest.py b/conftest.py index 27c398792..5a535c168 100644 --- a/conftest.py +++ b/conftest.py @@ -54,6 +54,9 @@ if not H2_ENABLED: if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") +if find_spec("pytest_codspeed") is None: + collect_ignore.append("tests/benchmarks") + def pytest_addoption(parser, pluginmanager): if pluginmanager.hasplugin("twisted"): diff --git a/docs/faq.rst b/docs/faq.rst index 00af9aa55..84afa2499 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -97,7 +97,7 @@ handler documentation. How can I scrape an item with attributes in different pages? ------------------------------------------------------------ -See :ref:`topics-request-response-ref-request-callback-arguments`. +See :ref:`callback-data`. How can I simulate a user login in my spider? --------------------------------------------- diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 406f848ae..cd65a2810 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -770,7 +770,7 @@ crawlers on top of it. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks -`. +`. Using spider arguments diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..257365380 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..87634cea9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 63c217e93..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement: Scrapy provides unified helpers for some of these examples: +.. autofunction:: scrapy.utils.asyncio.sleep .. autofunction:: scrapy.utils.asyncio.call_later .. autofunction:: scrapy.utils.asyncio.create_looping_call .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 93ef6a516..b9d9c5840 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -114,8 +114,8 @@ some usage help and the available commands:: scrapy [options] [args] Available commands: - crawl Run a spider fetch Fetch a URL using the Scrapy downloader + runspider Run a spider from a Python file, no project required [...] The first line will print the currently active project if you're inside a @@ -266,7 +266,9 @@ crawl * Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling using a spider. +Start crawling using the spider with the given :attr:`~scrapy.Spider.name`, +which must be one of those that :command:`list` reports. To run a spider from a +file instead, use :command:`runspider`. Supported options: @@ -580,8 +582,9 @@ runspider * Syntax: ``scrapy runspider `` * Requires project: *no* -Run a spider self-contained in a Python file, without having to create a -project. +Run the spider defined in the given Python file, without requiring a project. + +Supported options: the same as :command:`crawl`. Example usage:: @@ -674,6 +677,8 @@ Example: COMMANDS_MODULE = "mybot.commands" +.. note:: This is a :ref:`pre-crawler setting `. + .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 2ae7bc1af..0fd040f54 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.13 -- :class:`~scrapy.Request` callbacks. +- :class:`~scrapy.Request` :ref:`callbacks `, which may + also be defined as :term:`asynchronous generators `. - The :meth:`process_item` method of :ref:`item pipelines `. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a2b9604c6..d27a85d56 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -212,6 +212,16 @@ BaseItemExporter - ``None`` (all fields [2]_, default) + Fields are exported in declaration order, i.e. the order in which + they are defined in the :ref:`item class `. For + :class:`dict` items, which have no declared fields, the key order of + each item is used instead. + + .. versionchanged:: VERSION + Fields of non-\ :class:`dict` items used to be exported in the + order in which they had been populated, except in + :class:`CsvItemExporter`, which has always used declaration order. + - A list of fields: .. code-block:: python diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7e78ef353..a27468ea0 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,18 +136,10 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -The following stats are collected: - -* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). -* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). -* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). -* ``finish_reason``: the closing reason string (e.g. ``"finished"``, - ``"closespider_timeout"``). -* ``item_scraped_count``: total number of items that passed all pipelines. -* ``item_dropped_count``: total number of items dropped by a pipeline. -* ``item_dropped_reasons_count/``: per-exception drop count - (e.g. ``item_dropped_reasons_count/DropItem``). -* ``response_received_count``: total number of HTTP responses received. +The following stats are collected: :stat:`elapsed_time_seconds`, +:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`, +:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`, +:stat:`response_received_count`, :stat:`start_time`. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and: 1. sends a :signal:`memusage_warning_reached` signal when it exceeds :setting:`MEMUSAGE_WARNING_MB` -2. closes the spider with the `"memusage_exceeded"` reason when it exceeds +2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and @@ -215,7 +207,8 @@ An extension for debugging memory usage. It collects information about: :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The -info will be stored in the stats. +info will be stored in the :stat:`memdebug/gc_garbage_count` and +:stat:`memdebug/live_refs/{cls}` stats. .. _topics-extensions-ref-spiderstate: diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7e9192c5b..01745c4cc 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -218,12 +218,13 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html -You can also define a custom ACL, custom endpoint, and region name for exported -feeds using these settings: +You can also define a custom ACL, custom endpoint, region name and connection +pool size for exported feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` +- :setting:`AWS_MAX_POOL_CONNECTIONS` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 7f03c8eaf..9b0f25deb 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,9 +96,13 @@ Request serialization --------------------- For persistence to work, :class:`~scrapy.Request` objects must be -serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` -values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.Spider` class. +serializable with :mod:`pickle`, except for the :ref:`callback +` and :ref:`errback +` values passed to their ``__init__`` +method, which must be methods of the running :class:`~scrapy.Spider` class. + +Requests that cannot be serialized are kept in memory only: they are still +sent, but they are lost when the crawl is paused. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 503542249..10cc9d969 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +To reuse connections for as many files as you check or upload in parallel, set +:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly. + .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _Zenko CloudServer: https://www.zenko.io/cloudserver/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 608e7e1f2..ec9379185 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -205,10 +205,11 @@ Request objects Request metadata can also be accessed through the :attr:`~scrapy.http.Response.meta` attribute of a response. - To pass data from one spider callback to another, consider using - :attr:`cb_kwargs` instead. However, request metadata may be the right - choice in certain scenarios, such as to maintain some debugging data - across all follow-up requests (e.g. the source URL). + To pass your own data from one spider callback to another, use + :attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request + metadata may be the right choice in certain scenarios, such as to + maintain some debugging data across all follow-up requests (e.g. the + source URL). A common use of request metadata is to define request-specific parameters for Scrapy components (extensions, middlewares, etc.). For @@ -248,7 +249,7 @@ Request objects .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls]) @@ -256,7 +257,7 @@ Request objects given new values by whichever keyword arguments are specified. The :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also - :ref:`topics-request-response-ref-request-callback-arguments`. + :ref:`callback-data`. .. automethod:: from_curl @@ -347,160 +348,6 @@ Other functions related to requests .. autofunction:: scrapy.utils.httpobj.urlparse_cached -.. _topics-request-response-ref-request-callback-arguments: - -Passing additional data to callback functions ---------------------------------------------- - -The callback of a request is a function that will be called when the response -of that request is downloaded. The callback function will be called with the -downloaded :class:`Response` object as its first argument. - -Example: - -.. code-block:: python - - def parse_page1(self, response): - return scrapy.Request( - "http://www.example.com/some_page.html", callback=self.parse_page2 - ) - - - def parse_page2(self, response): - # this would log http://www.example.com/some_page.html - self.logger.info("Visited %s", response.url) - -In some cases you may be interested in passing arguments to those callback -functions so you can receive the arguments later, in the second callback. -The following example shows how to achieve this by using the -:attr:`.Request.cb_kwargs` attribute: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - cb_kwargs=dict(main_url=response.url), - ) - request.cb_kwargs["foo"] = "bar" # add more arguments for the callback - yield request - - - def parse_page2(self, response, main_url, foo): - yield dict( - main_url=main_url, - other_url=response.url, - foo=foo, - ) - -.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`.Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`.Request.meta` - for communication with components like middlewares and extensions. - -.. _topics-request-response-ref-errbacks: - -Using errbacks to catch exceptions in request processing --------------------------------------------------------- - -The errback of a request is a function that will be called when an exception -is raised while processing it. - -It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can -be used to track connection establishment timeouts, DNS errors etc. - -Here's an example spider logging all errors and catching some specific -errors if needed: - -.. code-block:: python - - import scrapy - - from scrapy.spidermiddlewares.httperror import HttpError - from twisted.internet.error import DNSLookupError - from twisted.internet.error import TimeoutError, TCPTimedOutError - - - class ErrbackSpider(scrapy.Spider): - name = "errback_example" - start_urls = [ - "http://www.httpbin.org/", # HTTP 200 expected - "http://www.httpbin.org/status/404", # Not found error - "http://www.httpbin.org/status/500", # server issue - "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "https://example.invalid/", # DNS error expected - ] - - async def start(self): - for u in self.start_urls: - yield scrapy.Request( - u, - callback=self.parse_httpbin, - errback=self.errback_httpbin, - dont_filter=True, - ) - - def parse_httpbin(self, response): - self.logger.info(f"Got successful response from {response.url}") - # do something useful here... - - def errback_httpbin(self, failure): - # log all failures - self.logger.error(repr(failure)) - - # in case you want to do something special for some errors, - # you may need the failure's type: - - if failure.check(HttpError): - # these exceptions come from HttpError spider middleware - # you can get the non-200 response - response = failure.value.response - self.logger.error("HttpError on %s", response.url) - - elif failure.check(DNSLookupError): - # this is the original request - request = failure.request - self.logger.error("DNSLookupError on %s", request.url) - - elif failure.check(TimeoutError, TCPTimedOutError): - request = failure.request - self.logger.error("TimeoutError on %s", request.url) - - -.. _errback-cb_kwargs: - -Accessing additional data in errback functions ----------------------------------------------- - -In case of a failure to process the request, you may be interested in -accessing arguments to the callback functions so you can process further -based on the arguments in the errback. The following example shows how to -achieve this by using ``Failure.request.cb_kwargs``: - -.. code-block:: python - - def parse(self, response): - request = scrapy.Request( - "http://www.example.com/index.html", - callback=self.parse_page2, - errback=self.errback_page2, - cb_kwargs=dict(main_url=response.url), - ) - yield request - - - def parse_page2(self, response, main_url): - pass - - - def errback_page2(self, failure): - yield dict( - main_url=failure.request.cb_kwargs["main_url"], - ) - - .. _request-fingerprints: Request fingerprints @@ -702,6 +549,319 @@ The following built-in Scrapy components have such restrictions: 45-character-long keys must be supported. +.. _callbacks: + +Callbacks +========= + +A callback is a function that Scrapy calls with the :class:`Response` of a +:class:`~scrapy.Request` once that request has been downloaded, so that you can +extract data from that response and generate additional requests to continue +the crawl: + +.. code-block:: python + + from scrapy import Request, Spider + + + class BookSpider(Spider): + name = "books" + + async def start(self): + yield Request("https://books.toscrape.com/", callback=self.parse_home) + + def parse_home(self, response): + for url in response.css("h3 a::attr(href)").getall(): + yield Request(response.urljoin(url), callback=self.parse_book) + + def parse_book(self, response): + yield {"title": response.css("h1::text").get()} + +Requests may also define an :ref:`errback `, which Scrapy calls +instead of the callback when an exception is raised while processing the +request or its response, e.g. a connection error or, by default, a non-2xx +response. + + +.. _callback-assignment: + +Assigning a callback to a request +--------------------------------- + +To assign a callback to a request, use the ``callback`` parameter of +:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse_home(response): ... + + + request = Request("https://books.toscrape.com/", callback=parse_home) + +Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to +``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider: + +.. code-block:: python + + request = Request("https://books.toscrape.com/") # Handled by parse() + +If a request is never meant to reach a spider callback, e.g. because a +:ref:`component ` sends it and handles its response itself, +assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it +instead, so that :ref:`downloader middlewares ` +can tell such requests apart. + +While :attr:`~scrapy.Request.callback` only accepts callables, some spider +classes let you also define a callback by name: both :attr:`CrawlSpider.rules +` and :attr:`SitemapSpider.sitemap_rules +` accept the name of a spider +method as a string. + + +.. _writing-callbacks: + +Writing a callback +------------------ + +Any callable can be a callback, as long as it takes the response as its first +positional parameter, and any :ref:`additional callback data ` +as keyword parameters. Spider methods are the most common choice, but plain +functions, lambda expressions and other callable objects work as well. + +.. note:: If you enable :ref:`job persistence ` through the + :setting:`JOBDIR` setting, callbacks must be methods of the running spider. + Requests with any other callback cannot be serialized, so they are kept in + memory only and lost when you pause the crawl. See + :ref:`request-serialization`. + +A callback can be: + +- A regular function: + + .. code-block:: python + + def parse(self, response): + return {"url": response.url} + +- A generator function: + + .. code-block:: python + + def parse(self, response): + yield {"url": response.url} + +- A coroutine function, i.e. defined with ``async def``: + + .. code-block:: python + + async def parse(self, response): + return {"url": response.url} + +- An asynchronous generator function: + + .. code-block:: python + + async def parse(self, response): + yield {"url": response.url} + +The last two allow using ``await``, ``async for`` and ``async with`` in your +callback. See :ref:`topics-coroutines`. + + +.. _callback-output: + +Callback output +--------------- + +A callback may return or yield any of the following: + +- ``None``, which does nothing. + + Callbacks that produce no output at all, e.g. callbacks that only log + information about the response, are perfectly valid. ``None`` values within + an iterable of callback output are ignored as well. + +- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and + eventually sends to its own callback. + +- An :ref:`item object `, which Scrapy sends to the + :ref:`item pipelines `. + + Any object that is neither ``None`` nor a :class:`~scrapy.Request` object + is treated as an item. + +- An iterable of any of the values above, e.g. a list or, more commonly, a + generator. + + :term:`Asynchronous iterables `, e.g. an + :term:`asynchronous generator`, are also supported. + +.. note:: When a callback *returns* an object, Scrapy iterates that object if + it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`, + :class:`str` and :class:`bytes` objects, which are always handled as single + items. + +.. note:: In a generator callback, a ``return`` statement with a value does not + produce any output, since such a value is not part of what the generator + yields. Scrapy logs a warning when it detects such a callback, see + :setting:`WARN_ON_GENERATOR_RETURN_VALUE`. + +Before Scrapy acts on the output of a callback, that output goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of your :ref:`spider middlewares `, which may modify +it or drop part of it. + +If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the +request is *not* called. The exception goes through the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` +method of your spider middlewares instead and, unless one of them handles it, +Scrapy logs it and sends the :signal:`spider_error` signal. + + +.. _callback-data: +.. _topics-request-response-ref-request-callback-arguments: + +Passing additional data to callback functions +--------------------------------------------- + +In some cases you may be interested in passing data to a callback in addition +to the response, e.g. data extracted from the response that triggered the +request. The following example shows how to achieve this by using the +:attr:`.Request.cb_kwargs` attribute: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + cb_kwargs=dict(main_url=response.url), + ) + request.cb_kwargs["foo"] = "bar" # add more arguments for the callback + yield request + + + def parse_page2(self, response, main_url, foo): + yield dict( + main_url=main_url, + other_url=response.url, + foo=foo, + ) + +:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a +callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components +`, such as middlewares and extensions. + +.. _errbacks: +.. _topics-request-response-ref-errbacks: + +Errbacks +======== + +The errback of a request is a function that will be called when an exception +is raised while processing it. + +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. + +Here's an example spider logging all errors and catching some specific +errors if needed: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.spidermiddlewares.httperror import HttpError + from twisted.internet.error import DNSLookupError + from twisted.internet.error import TimeoutError, TCPTimedOutError + + + class ErrbackSpider(Spider): + name = "errback_example" + start_urls = [ + "http://www.httpbin.org/", # HTTP 200 expected + "http://www.httpbin.org/status/404", # Not found error + "http://www.httpbin.org/status/500", # server issue + "http://www.httpbin.org:12345/", # non-responding host, timeout expected + "https://example.invalid/", # DNS error expected + ] + + async def start(self): + for u in self.start_urls: + yield Request( + u, + callback=self.parse_httpbin, + errback=self.errback_httpbin, + dont_filter=True, + ) + + def parse_httpbin(self, response): + self.logger.info(f"Got successful response from {response.url}") + # do something useful here... + + def errback_httpbin(self, failure): + # log all failures + self.logger.error(repr(failure)) + + # in case you want to do something special for some errors, + # you may need the failure's type: + + if failure.check(HttpError): + # these exceptions come from HttpError spider middleware + # you can get the non-200 response + response = failure.value.response + self.logger.error("HttpError on %s", response.url) + + elif failure.check(DNSLookupError): + # this is the original request + request = failure.request + self.logger.error("DNSLookupError on %s", request.url) + + elif failure.check(TimeoutError, TCPTimedOutError): + request = failure.request + self.logger.error("TimeoutError on %s", request.url) + + +.. _errback-cb_kwargs: + +Accessing additional data in errback functions +---------------------------------------------- + +In case of a failure to process the request, you may be interested in +accessing arguments to the callback functions so you can process further +based on the arguments in the errback. The following example shows how to +achieve this by using ``Failure.request.cb_kwargs``: + +.. code-block:: python + + from scrapy import Request + + + def parse(self, response): + request = Request( + "http://www.example.com/index.html", + callback=self.parse_page2, + errback=self.errback_page2, + cb_kwargs=dict(main_url=response.url), + ) + yield request + + + def parse_page2(self, response, main_url): + pass + + + def errback_page2(self, failure): + yield dict( + main_url=failure.request.cb_kwargs["main_url"], + ) + + .. _topics-request-meta: Request.meta special keys diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 639f580cf..218128caa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider `. These settings are: -- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`ADDONS` +- :setting:`COMMANDS_MODULE` +- :setting:`FORCE_CRAWLER_PROCESS` - :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding spider loader class, e.g. :setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. +- :setting:`TWISTED_REACTOR_ENABLED` + +:setting:`ADDONS` is a special case: it can be set from a spider, but the +``update_pre_crawler_settings()`` method of :ref:`add-ons ` +enabled that way is not called. + +:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a +:ref:`command that needs a CrawlerProcess `, +since its project-level value determines the crawler process class. .. _reactor-settings: @@ -409,6 +420,9 @@ Default: ``{}`` A dict containing paths to the add-ons enabled in your project and their priorities. For more information, see :ref:`topics-addons`. +.. note:: This is a :ref:`pre-crawler setting `, with a + caveat described in that section. + .. setting:: ASYNCIO_EVENT_LOOP ASYNCIO_EVENT_LOOP @@ -458,6 +472,26 @@ Default: ``None`` Endpoint URL used for S3-like storage, for example Minio or s3.scality. +.. setting:: AWS_MAX_POOL_CONNECTIONS + +AWS_MAX_POOL_CONNECTIONS +------------------------ + +.. versionadded:: VERSION + +Default: ``None`` + +Maximum number of connections that AWS clients, such as those of the +:ref:`S3 feed storage backend ` and of the +:ref:`S3 media pipeline storage backend `, keep in their +connection pool. + +If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used. + +Values lower than the number of parallel AWS calls do not limit those calls, but +their connections are closed instead of reused, which hurts performance, and +``Connection pool is full, discarding connection`` warnings are logged. + .. setting:: AWS_REGION_NAME AWS_REGION_NAME @@ -541,7 +575,7 @@ CONCURRENT_REQUESTS Default: ``16`` The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +performed by the Scrapy downloader. Use ``0`` for no limit. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -1382,6 +1416,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a non-default value in :ref:`per-spider settings `. +.. note:: This is a :ref:`pre-crawler setting `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -1835,7 +1871,8 @@ Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. This currently logs (only once) if the requests cannot be serialized to disk. -Stats counter (``scheduler/unserializable``) tracks the number of times this happens. +The :stat:`scheduler/unserializable` stat tracks the number of times this +happens. Example entry in logs:: diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 083c2294d..22f3960b4 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -503,6 +503,27 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + .. versionadded:: VERSION + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a :file:`robots.txt` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed :file:`robots.txt` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the :file:`robots.txt` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b792d4be5..fba64cd49 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -4,43 +4,31 @@ Spiders ======= -Spiders are classes which define how a certain site (or a group of sites) will be -scraped, including how to perform the crawl (i.e. follow links) and how to -extract structured data from their pages (i.e. scraping items). In other words, -Spiders are the place where you define the custom behaviour for crawling and -parsing pages for a particular site (or, in some cases, a group of sites). +Spiders are classes that define how a site, or a group of sites, is scraped: +which requests to send, and how to parse their responses to extract data and to +send additional requests. -For spiders, the scraping cycle goes through something like this: +A crawl goes as follows: -1. You start by generating the initial requests to crawl the first URLs, and - specify a callback function to be called with the response downloaded from - those requests. +1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to + get the initial requests. By default, that method yields a + :class:`~scrapy.Request` object for each URL in + :attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as + :ref:`callback `. - The first requests to perform are obtained by iterating the - :meth:`~scrapy.Spider.start` method, which by default yields a - :class:`~scrapy.Request` object for each URL in the - :attr:`~scrapy.Spider.start_urls` spider attribute, with the - :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` - function to handle each :class:`~scrapy.http.Response`. +2. Scrapy downloads each request and calls its callback with the resulting + :class:`~scrapy.http.Response`. -2. In the callback function, you parse the response (web page) and return - :ref:`item objects `, - :class:`~scrapy.Request` objects, or an iterable of these objects. - Those Requests will also contain a callback (maybe - the same) and will then be downloaded by Scrapy and then their - response handled by the specified callback. +3. Callbacks parse the response, typically using :ref:`topics-selectors`, and + return or yield :ref:`item objects ` with the extracted data + and :class:`~scrapy.Request` objects to continue the crawl, which go back + to step 2. See :ref:`callback-output`. -3. In callback functions, you parse the page contents, typically using - :ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever - mechanism you prefer) and generate items with the parsed data. +4. Items go through :ref:`item pipelines `, and are + usually stored through :ref:`topics-feed-exports`. -4. Finally, the items returned from the spider will be typically persisted to a - database (in some :ref:`Item Pipeline `) or written to - a file using :ref:`topics-feed-exports`. - -Even though this cycle applies (more or less) to any kind of spider, there are -different kinds of default spiders bundled into Scrapy for different purposes. -We will talk about those types here. +Scrapy includes different spider classes for different purposes, described +below. .. _topics-spiders-ref: @@ -191,22 +179,7 @@ scrapy.Spider .. automethod:: start - .. method:: parse(response) - - This is the default callback used by Scrapy to process downloaded - responses, when their requests don't specify a callback. - - The ``parse`` method is in charge of processing the response and returning - scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`~scrapy.Spider` class. - - This method, as well as any other Request callback, must return a - :class:`~scrapy.Request` object, an :ref:`item object `, an - iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects - `, or ``None``. - - :param response: the response to parse - :type response: :class:`~scrapy.http.Response` + .. automethod:: parse .. method:: closed(reason) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index e6d97625e..142b7918b 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -22,6 +22,8 @@ using the Stats Collector from. Another feature of the Stats Collector is that it's very efficient (when enabled) and almost unnoticeable when disabled. +See :ref:`topics-stats-reference` below for the stats that Scrapy sets. + .. _topics-stats-usecases: Common Stats Collector uses @@ -102,3 +104,642 @@ DummyStatsCollector ------------------- .. autoclass:: DummyStatsCollector + +.. _topics-stats-reference: + +Built-in stats reference +======================== + +Scrapy sets the following :ref:`stats `. Components other than +those built into Scrapy may set additional stats; see their documentation. + +Stat keys that contain a ``{placeholder}`` below stand for a family of stats, +one per actual value of the placeholder. + +.. note:: Most stats are set by a specific :ref:`component + `, and are only present if that component is enabled and + its code path is reached. A stat that is missing from + :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is + equivalent to a counter of 0. + +.. stat:: downloader/exception_count + +``downloader/exception_count`` + Number of exceptions raised while downloading requests. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/exception_type_count/{exception_type} + +``downloader/exception_type_count/{exception_type}`` + Number of exceptions raised while downloading requests, per exception type, + where ``{exception_type}`` is the import path of the exception class, e.g. + ``twisted.internet.error.DNSLookupError``. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_bytes + +``downloader/request_bytes`` + Total size, in bytes, of the requests sent, counting the request line, the + headers and the body. As with :stat:`downloader/request_count`, requests + served from the cache are also counted. + + It is an approximation, reconstructed from each :class:`~scrapy.Request` + object instead of measured on the wire, so it does not account for the + actual bytes that the :ref:`download handler + ` sends, e.g. transport-level overhead. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_count + +``downloader/request_count`` + Number of requests sent. + + Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache are also counted, even though they are never sent, + because it handles requests after + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_method_count/{method} + +``downloader/request_method_count/{method}`` + Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with + :stat:`downloader/request_count`, requests served from the cache are also + counted. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_bytes + +``downloader/response_bytes`` + Total size, in bytes, of the responses received, counting the status line, + the headers and the body. It covers the same responses as + :stat:`downloader/response_count`. + + The body is counted as received, i.e. still compressed for responses that + used ``Content-Encoding``, because + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles + responses before + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + decompresses them. See :stat:`httpcompression/response_bytes` for + decompressed sizes. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_count + +``downloader/response_count`` + Number of responses received. + + It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache, even though they do not come from the network, and + responses that a downloader middleware consumes before they reach your + spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Compare with :stat:`response_received_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_status_count/{status_code} + +``downloader/response_status_count/{status_code}`` + Number of responses received, per HTTP status code, e.g. ``200`` or + ``404``. It covers the same responses as :stat:`downloader/response_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: dupefilter/filtered + +``dupefilter/filtered`` + Number of requests dropped as duplicates. + + Set by :class:`~scrapy.dupefilters.RFPDupeFilter`. + +.. stat:: elapsed_time_seconds + +``elapsed_time_seconds`` + Time, as a :class:`float`, in seconds, between the :signal:`spider_opened` + and the :signal:`spider_closed` signals. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: feedexport/failed_count/{storage} + +``feedexport/failed_count/{storage}`` + Number of :ref:`feeds ` that could not be stored, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: feedexport/success_count/{storage} + +``feedexport/success_count/{storage}`` + Number of :ref:`feeds ` stored successfully, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: file_count + +``file_count`` + Number of files handled by the :ref:`media pipelines + `. + +.. stat:: file_status_count/{status} + +``file_status_count/{status}`` + Number of files handled by the :ref:`media pipelines + `, per status, where ``{status}`` is one of: + + - ``downloaded``: the file was downloaded. + + - ``cached``: the file came from the + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + cache. + + - ``uptodate``: the file was already in the storage backend and had not + :ref:`expired `, so it was not downloaded again. + +.. stat:: finish_reason + +``finish_reason`` + String indicating why the crawl finished. It matches the *reason* argument + of the :signal:`spider_closed` signal. + + Scrapy uses the following reasons: + + - ``cancelled``: the spider was closed without a more specific reason, + e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without + one. + + - ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`. + + - ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`. + + - ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`. + + - ``closespider_pagecount_no_item``: see + :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`. + + - ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`. + + - ``closespider_timeout_no_item``: see + :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + + - ``finished``: the spider became idle with no pending requests, i.e. it + finished normally. + + - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such + as ``SIGINT`` (:kbd:`Ctrl-C`). + + Third-party components and your own code may use any other reason, e.g. by + raising :exc:`~scrapy.exceptions.CloseSpider` with it. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: finish_time + +``finish_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_closed` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: httpcache/errorrecovery + +``httpcache/errorrecovery`` + Number of times that a stale cached response was used because downloading a + fresh response raised an exception. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/firsthand + +``httpcache/firsthand`` + Number of responses that were downloaded without a matching cache entry to + validate against, i.e. responses for requests counted in + :stat:`httpcache/miss`. + + It is lower than :stat:`httpcache/miss` when some of those requests yield + no response, either because they are dropped (see + :stat:`httpcache/ignore`) or because their download fails. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/hit + +``httpcache/hit`` + Number of requests served from the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/ignore + +``httpcache/ignore`` + Number of requests dropped because they were not in the cache and + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/invalidate + +``httpcache/invalidate`` + Number of times that a cached response failed validation and was replaced + with a freshly downloaded response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/miss + +``httpcache/miss`` + Number of requests for which no cache entry could be read, either because + there was none or because reading it failed, in which case the request is + also counted in :stat:`httpcache/retrieve_error`. Those requests are + downloaded (see :stat:`httpcache/firsthand`), or dropped if + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see + :stat:`httpcache/ignore`). + + Requests with a stale cache entry are not counted here; see + :stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/retrieve_error + +``httpcache/retrieve_error`` + Number of cache entries that could not be read, and hence were treated as + cache misses. Those requests are also counted in :stat:`httpcache/miss`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/revalidate + +``httpcache/revalidate`` + Number of times that a cached response was successfully validated against + the target server, and hence used instead of the fresh response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/store + +``httpcache/store`` + Number of responses stored in the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/uncacheable + +``httpcache/uncacheable`` + Number of responses not stored in the cache because the + :setting:`HTTPCACHE_POLICY` did not allow it. + + Every response considered for caching is counted either here or in + :stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable`` + equals ``httpcache/firsthand + httpcache/invalidate``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcompression/response_bytes + +``httpcompression/response_bytes`` + Total size, in bytes, of decompressed response bodies, counting only the + body and only responses that were actually decompressed. Compare with + :stat:`downloader/response_bytes`. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httpcompression/response_count + +``httpcompression/response_count`` + Number of decompressed responses. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httperror/response_ignored_count + +``httperror/response_ignored_count`` + Number of responses dropped because of their HTTP status code. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: httperror/response_ignored_status_count/{status_code} + +``httperror/response_ignored_status_count/{status_code}`` + Number of responses dropped because of their HTTP status code, per HTTP + status code, e.g. ``404``. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: item_dropped_count + +``item_dropped_count`` + Number of items dropped by an :ref:`item pipeline + `, i.e. number of times that the + :signal:`item_dropped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_dropped_reasons_count/{exception} + +``item_dropped_reasons_count/{exception}`` + Number of items dropped, per exception, where ``{exception}`` is the class + name of the exception that caused the item to be dropped. + + Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and + each one is counted under its own class name, e.g. + ``item_dropped_reasons_count/DropItem`` for + :exc:`~scrapy.exceptions.DropItem` itself and + ``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of + it. Any other exception raised by an :ref:`item pipeline + ` triggers the :signal:`item_error` signal instead of + :signal:`item_dropped`, and is not counted here or in + :stat:`item_dropped_count`. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_scraped_count + +``item_scraped_count`` + Number of items that passed all :ref:`item pipelines + `, i.e. number of times that the + :signal:`item_scraped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: items_per_minute + +``items_per_minute`` + Average number of items scraped per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: log_count/{level} + +``log_count/{level}`` + Number of log messages, per logging level name, e.g. ``INFO`` or + ``WARNING``. + + Only messages that the :setting:`LOG_LEVEL` setting allows are counted. + + Set by :class:`~scrapy.extensions.logcount.LogCount`. + +.. stat:: memdebug/gc_garbage_count + +``memdebug/gc_garbage_count`` + Number of objects in :data:`gc.garbage` when the spider is closed. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memdebug/live_refs/{cls} + +``memdebug/live_refs/{cls}`` + Number of live objects of class ``{cls}`` when the spider is closed, as + reported by :ref:`trackref `, e.g. + ``memdebug/live_refs/HtmlResponse``. + + Only set for classes with at least 1 live object. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memusage/limit_reached + +``memusage/limit_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also + stops the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/max + +``memusage/max`` + Maximum peak memory usage, in bytes, observed during the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/startup + +``memusage/startup`` + Peak memory usage, in bytes, when the engine started. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/warning_reached + +``memusage/warning_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: offsite/domains + +``offsite/domains`` + Number of distinct domains for which at least 1 request was dropped for + being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: offsite/filtered + +``offsite/filtered`` + Number of requests dropped for being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: request_depth_count/{depth} + +``request_depth_count/{depth}`` + Number of requests scheduled at depth ``{depth}``, e.g. + ``request_depth_count/2``. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which + requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat. + +.. stat:: request_depth_max + +``request_depth_max`` + Maximum depth reached. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + +.. stat:: response_received_count + +``response_received_count`` + Number of responses received, i.e. number of times that the + :signal:`response_received` signal was sent. + + Unlike :stat:`downloader/response_count`, it does not count responses that + a downloader middleware consumes before they reach the engine, e.g. + redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: responses_per_minute + +``responses_per_minute`` + Average number of responses received per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: retry/count + +``retry/count`` + Number of requests retried. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/max_reached + +``retry/max_reached`` + Number of requests that were not retried because they had already been + retried :setting:`RETRY_TIMES` times. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/reason_count/{reason} + +``retry/reason_count/{reason}`` + Number of requests retried, per reason, e.g. + ``retry/reason_count/twisted.internet.error.TimeoutError`` or + ``retry/reason_count/504 Gateway Time-out``. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. note:: Code calling + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a + custom *stats_base_key*, in which case ``retry`` is replaced with that key + in the 3 stats above. + +.. stat:: robotstxt/exception_count/{exception_type} + +``robotstxt/exception_count/{exception_type}`` + Number of exceptions raised while downloading ``robots.txt`` files, per + exception type, where ``{exception_type}`` is the string representation of + the exception class, e.g. ````. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/forbidden + +``robotstxt/forbidden`` + Number of requests dropped for being disallowed by ``robots.txt``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/request_count + +``robotstxt/request_count`` + Number of ``robots.txt`` files requested, i.e. 1 per network location for + which at least 1 request was sent. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_count + +``robotstxt/response_count`` + Number of ``robots.txt`` responses received. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_status_count/{status_code} + +``robotstxt/response_status_count/{status_code}`` + Number of ``robots.txt`` responses received, per HTTP status code, e.g. + ``404``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: scheduler/dequeued + +``scheduler/dequeued`` + Number of requests read from the :ref:`scheduler `. + +.. stat:: scheduler/dequeued/disk + +``scheduler/dequeued/disk`` + Number of requests read from the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/dequeued/memory + +``scheduler/dequeued/memory`` + Number of requests read from the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued + +``scheduler/enqueued`` + Number of requests stored into the :ref:`scheduler `. + +.. stat:: scheduler/enqueued/disk + +``scheduler/enqueued/disk`` + Number of requests stored into the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued/memory + +``scheduler/enqueued/memory`` + Number of requests stored into the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/unserializable + +``scheduler/unserializable`` + Number of requests that could not be stored into the disk queue of the + :ref:`scheduler ` because they could not be + :ref:`serialized `, and hence were stored into the + memory queue instead. + +.. stat:: spider_exceptions/count + +``spider_exceptions/count`` + Number of unhandled exceptions raised by spider callbacks. + + Set by the :ref:`scraper `. + +.. stat:: spider_exceptions/{exception} + +``spider_exceptions/{exception}`` + Number of unhandled exceptions raised by spider callbacks, per exception, + where ``{exception}`` is the class name of the exception, e.g. + ``spider_exceptions/ValueError``. + + Set by the :ref:`scraper `. + +.. stat:: start_time + +``start_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_opened` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: urllength/request_ignored_count + +``urllength/request_ignored_count`` + Number of requests dropped for having a URL longer than + :setting:`URLLENGTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`. diff --git a/pyproject.toml b/pyproject.toml index 576a42e5c..1cbd39946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ brotli = [ gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] -ipython = ["ipython>=7.1.0"] -ptpython = ["ptpython>=2.0.1"] +ipython = ["ipython>=8.15.0"] +ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 866ba9f6b..4e086e057 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a spider" + return "Run a spider of the current project, by name" def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4277232c3..52f9cd4b0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str: def extract_domain(url: str) -> str: """Extract domain name from URL string""" - o = urlparse(url) - if o.scheme == "" and o.netloc == "": - o = urlparse("//" + url.lstrip("/")) - return o.netloc + return urlparse(url).netloc def verify_url_scheme(url: str) -> str: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 93194ded7..51caed57f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): spider: Spider | None = None items: ClassVar[dict[int, list[Any]]] = {} requests: ClassVar[dict[int, list[Request]]] = {} - spidercls: type[Spider] | None + spidercls: type[Spider] | None = None first_response = None diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 0b9036457..9cdb393ab 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand): return "[options] " def short_desc(self) -> str: - return "Run a self-contained spider (without creating a project)" + return "Run a spider from a Python file, no project required" def long_desc(self) -> str: return "Run the spider defined in the given file" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..eb2079d0c 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -138,7 +138,8 @@ class Downloader: self.active.remove(request) def needs_backout(self) -> bool: - return len(self.active) >= self.total_concurrency + # A total concurrency of 0 means no limit. + return 0 < self.total_concurrency <= len(self.active) @_warn_spider_arg def _get_slot( diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index d5a4e9fcd..8bbffb233 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base): self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( - # hard limit on simultaneous connections - max_connections=self._pool_size_total, + # hard limit on simultaneous connections (None for no limit, which + # is what a CONCURRENT_REQUESTS of 0 means) + max_connections=self._pool_size_total or None, # total number of idle connections in the pool (extra ones are closed) - max_keepalive_connections=self._pool_size_total, + max_keepalive_connections=self._pool_size_total or None, ) self._default_client: httpx.AsyncClient = self._make_client() diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 42c517222..a511b0c7b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -366,8 +366,8 @@ class Scheduler(BaseScheduler): Unless the received request is filtered out by the Dupefilter, attempt to push it into the disk queue, falling back to pushing it into the memory queue. - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/enqueued`, + :stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`. Return ``True`` if the request was stored successfully, ``False`` otherwise. """ @@ -390,8 +390,8 @@ class Scheduler(BaseScheduler): falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/dequeued`, + :stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`. """ request: Request | None = self.mqs.pop() assert self.stats is not None diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 5a11df833..ea600d1a8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -74,6 +74,17 @@ class BaseItemExporter(ABC): def finish_exporting(self) -> None: # noqa: B027 pass + @staticmethod + def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]: + """Return the populated field names of *adapter*, in declaration order. + + Populated fields that are not declared, which some item types allow, + come last, in item order. + """ + populated = set(adapter.keys()) + declared = (name for name in adapter.field_names() if name in populated) + return dict.fromkeys([*declared, *adapter.keys()]) + def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: @@ -86,7 +97,11 @@ class BaseItemExporter(ABC): include_empty = self.export_empty_fields if self.fields_to_export is None: - field_iter = item.field_names() if include_empty else item.keys() + field_iter = ( + item.field_names() + if include_empty + else self._get_populated_field_names(item) + ) elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a4362b182..9cb792e30 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -119,7 +119,7 @@ class CloseSpider: self.task = None if self.task_no_item: - if self.task_no_item.running: + if self.task_no_item.running: # pragma: no branch self.task_no_item.stop() self.task_no_item = None diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c2997921d..118462bc9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -28,6 +28,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.asyncio import is_asyncio_available, run_in_thread +from scrapy.utils.boto import _get_max_pool_connections from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage): feed_options: dict[str, Any] | None = None, session_token: str | None = None, region_name: str | None = None, + max_pool_connections: int | None = None, ): try: import boto3.session # noqa: PLC0415 except ImportError: raise NotConfigured("missing boto3 library") from None + from botocore.config import Config # noqa: PLC0415 + u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name + self.max_pool_connections: int | None = max_pool_connections boto3_session = boto3.session.Session() self.s3_client = boto3_session.client( @@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage): aws_session_token=self.session_token, endpoint_url=self.endpoint_url, region_name=self.region_name, + config=( + Config(max_pool_connections=self.max_pool_connections) + if self.max_pool_connections is not None + else None + ), ) if feed_options and feed_options.get("overwrite", True) is False: @@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage): acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, region_name=crawler.settings["AWS_REGION_NAME"] or None, + max_pool_connections=_get_max_pool_connections(crawler.settings), feed_options=feed_options, ) @@ -454,7 +465,7 @@ class FeedSlot: ) def finish_exporting(self) -> None: - if self._exporting: + if self._exporting: # pragma: no branch assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -542,7 +553,7 @@ class FeedExporter: for slot in self.slots: self._schedule_slot_close(slot, spider) - if self._pending_close_tasks: + if self._pending_close_tasks: # pragma: no branch if is_asyncio_available(): await asyncio.wait( cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)) diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py index e6d51a7d8..fcce64438 100644 --- a/scrapy/extensions/logcount.py +++ b/scrapy/extensions/logcount.py @@ -20,7 +20,7 @@ class LogCount: """Install a log handler that counts log messages by level. The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. - The counts are stored in stats as ``log_count/``. + The counts are stored in the :stat:`log_count/{level}` stat. .. versionadded:: 2.14 """ diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 1444c8941..e0e289ce8 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -94,7 +94,7 @@ class MemoryUsage: def engine_stopped(self) -> None: for tsk in self.tasks: - if tsk.running: + if tsk.running: # pragma: no branch tsk.stop() def update(self) -> None: diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index cd35c8165..cbcc8b70e 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -38,7 +38,6 @@ class PeriodicLog: ): self.stats: StatsCollector = stats self.interval: float = interval - self.multiplier: float = 60.0 / self.interval self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) @@ -165,5 +164,5 @@ class PeriodicLog: def spider_closed(self, spider: Spider, reason: str) -> None: self.log() - if self.task and self.task.running: + if self.task and self.task.running: # pragma: no branch self.task.stop() diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 68847283a..7d67bb6d7 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -169,7 +169,8 @@ class Request(object_ref): #: #: The callable must expect the response as its first parameter, and #: support any additional keyword arguments set through - #: :attr:`cb_kwargs`. + #: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and + #: :ref:`callback-output`. #: #: In addition to an arbitrary callable, the following values are also #: supported: @@ -190,8 +191,7 @@ class Request(object_ref): #: raises exceptions for non-2xx responses by default, sending them #: to the :attr:`errback` instead. #: - #: .. seealso:: - #: :ref:`topics-request-response-ref-request-callback-arguments` + #: .. seealso:: :ref:`callbacks` self.callback: CallbackT | None = callback #: :class:`~collections.abc.Callable` to handle exceptions raised @@ -200,7 +200,7 @@ class Request(object_ref): #: The callable must expect a :exc:`~twisted.python.failure.Failure` as #: its first parameter. #: - #: .. seealso:: :ref:`topics-request-response-ref-errbacks` + #: .. seealso:: :ref:`errbacks` self.errback: Callable[[Failure], Any] | None = errback self._cookies: CookiesT | None = cookies or None diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8e8082332..55a3676e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.pipelines.media import ( _MediaRequestFiltered, ) from scrapy.utils.asyncio import run_in_thread -from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -164,6 +164,9 @@ class S3FilesStore: AWS_REGION_NAME = None AWS_USE_SSL = None AWS_VERIFY = None + # Overridden from settings.AWS_MAX_POOL_CONNECTIONS in + # FilesPipeline.from_crawler(); None means the botocore default + AWS_MAX_POOL_CONNECTIONS: int | None = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() HEADERS: ClassVar[dict[str, str]] = { @@ -174,7 +177,13 @@ class S3FilesStore: if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session # noqa: PLC0415 + from botocore.config import Config # noqa: PLC0415 + config = ( + Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS) + if self.AWS_MAX_POOL_CONNECTIONS is not None + else None + ) session = botocore.session.get_session() self.s3_client = session.create_client( "s3", @@ -185,6 +194,7 @@ class S3FilesStore: region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, verify=self.AWS_VERIFY, + config=config, ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") @@ -522,6 +532,7 @@ class FilesPipeline(MediaPipeline): s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings) s3store.POLICY = settings["FILES_STORE_S3_ACL"] gcs_store: type[GCSFilesStore] = cast( diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..b54011784 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + .. versionadded:: VERSION + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser): url = to_unicode(url) return cast("bool", self.rp.is_allowed(user_agent, url)) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.get_crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 993f2436d..a44b36c8a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -28,6 +28,7 @@ __all__ = [ "AUTOTHROTTLE_TARGET_CONCURRENCY", "AWS_ACCESS_KEY_ID", "AWS_ENDPOINT_URL", + "AWS_MAX_POOL_CONNECTIONS", "AWS_REGION_NAME", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", @@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None +AWS_MAX_POOL_CONNECTIONS = None AWS_REGION_NAME = None AWS_SESSION_TOKEN = None AWS_USE_SSL = None diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 02dfa2ac6..6244e3264 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -143,6 +143,22 @@ class Spider(object_ref): else: def parse(self, response: Response, **kwargs: Any) -> Any: + """Process *response*, i.e. extract data from it and generate new + requests. + + This is the default :ref:`callback `: Scrapy uses + it for the response to any request that does not define a + :attr:`~scrapy.Request.callback`, such as the requests that + :meth:`start` yields by default. + + Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as + keyword parameters. + + Spiders must define this method, unless every request that they + send defines a callback. + + See :ref:`callback-output` about the supported return values. + """ raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..7c7697f56 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -293,6 +293,24 @@ class CallLaterResult: self._delayed_call = None +async def sleep(seconds: float) -> None: + """Sleep for *seconds*. + + .. versionadded:: VERSION + + This uses either :func:`asyncio.sleep` or + :func:`~twisted.internet.task.deferLater`, depending on whether asyncio + support is available. + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + return + + from twisted.internet import reactor + + await deferLater(reactor, seconds) + + async def run_in_thread( func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs ) -> _T: diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 2a77ee2ac..76ee0e7ec 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,22 @@ """Boto/botocore helpers""" +from __future__ import annotations + from importlib.util import find_spec +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def is_botocore_available() -> bool: return find_spec("botocore") is not None + + +def _get_max_pool_connections(settings: BaseSettings) -> int: + """Return the maximum number of connections that AWS clients may keep in + their connection pool. + """ + return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint( + "REACTOR_THREADPOOL_MAXSIZE" + ) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 31a4bb32f..23b4401e8 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,8 +1,9 @@ from __future__ import annotations +import asyncio import code from collections.abc import Callable -from functools import wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -16,16 +17,8 @@ def _embed_ipython_shell( namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" - try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 - from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 - except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 - InteractiveShellEmbed, - ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 - load_default_config, - ) + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 @wraps(_embed_ipython_shell) def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: @@ -38,6 +31,19 @@ def _embed_ipython_shell( shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config ) + # If an asyncio event loop is already running in this thread, e.g. when + # inspect_response() is called from a spider callback while using the + # asyncio reactor, prompt_toolkit cannot run its own event loop here, so + # ask it to run the prompt in a separate thread instead. pt_app is None + # when IPython falls back to its simple prompt, which needs no event loop. + # See https://github.com/scrapy/scrapy/issues/5447 + if (pt_app := getattr(shell, "pt_app", None)) is not None: + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + pt_app.prompt = partial(pt_app.prompt, in_thread=True) shell() return wrapper diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4960dc27a..2924c81f9 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncio import run_in_thread from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.python import _signature if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -109,7 +110,7 @@ def _warn_spider_arg( ): """Decorator to warn if a ``spider`` argument is passed to a function.""" - sig = inspect.signature(func) + sig = _signature(func) def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: bound = sig.bind(*args, **kwargs) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..7c6235f29 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, sleep from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None: """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ - if is_asyncio_available(): - await asyncio.sleep(_DEFER_DELAY) - else: - from twisted.internet import reactor - - d: Deferred[None] = Deferred() - reactor.callLater(_DEFER_DELAY, d.callback, None) - await d + await sleep(_DEFER_DELAY) def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8a7517c1d..40fc05257 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) +# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature() +# can raise NameError for names imported only under TYPE_CHECKING. We only need +# parameter names, kinds and defaults, so leave such annotations as ForwardRefs. +if sys.version_info >= (3, 14): + from annotationlib import Format + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func, annotation_format=Format.FORWARDREF) + +else: + + def _signature(func: Callable[..., Any]) -> inspect.Signature: + return inspect.signature(func) + + def get_func_args_dict( func: Callable[..., Any], stripself: bool = False ) -> Mapping[str, inspect.Parameter]: """Return the argument dict of a callable object. + Annotations are not evaluated, so on Python 3.14 and later the ``annotation`` + attribute of the returned parameters may be a ``ForwardRef`` instead of the + resolved type. + .. versionadded:: 2.14 """ if not callable(func): @@ -190,7 +209,7 @@ def get_func_args_dict( args: Mapping[str, inspect.Parameter] try: - sig = inspect.signature(func) + sig = _signature(func) except ValueError: return {} diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 000000000..7b5ca0cb9 --- /dev/null +++ b/tests/benchmarks/__init__.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy import Spider + from scrapy.crawler import Crawler + + +def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler: + """Run a crawl to completion and return its crawler. + + Unlike the rest of the test suite, benchmarks run without ``pytest-twisted`` + and drive the reactor themselves, since the code being measured must be + callable synchronously by ``pytest-codspeed``. + """ + from twisted.internet import reactor + + crawler = get_crawler(spidercls, settings) + result: list[Any] = [] + crawler.crawl(**kwargs).addBoth(result.append) + while not result: + reactor.iterate(0.001) + if isinstance(result[0], BaseException): + raise result[0] + return crawler diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 000000000..55356083d --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.utils.reactor import install_reactor + +if TYPE_CHECKING: + from collections.abc import Generator + + +@pytest.fixture(scope="session", autouse=True) +def running_reactor() -> Generator[None]: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + from twisted.internet import reactor + + # Marks the reactor as running without blocking, so that crawls can be + # driven with reactor.iterate(), see tests.benchmarks.crawl(). + reactor.startRunning(installSignalHandlers=False) + + yield + + reactor.stop() + # Lets the shutdown event triggers run, e.g. to join the thread pool. + reactor.iterate(0) diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.py new file mode 100644 index 000000000..0fdfe742b --- /dev/null +++ b/tests/benchmarks/test_crawl.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urlencode + +import pytest + +from scrapy import Field, Item, Request, Spider +from scrapy.linkextractors import LinkExtractor +from tests.benchmarks import crawl + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found] + + from scrapy.http import Response + from tests.mockserver.http import MockServer + +pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed") + +PAGES = 100 +LINKS_PER_PAGE = 5 + + +class _Page(Item): + url = Field() + anchors = Field() + + +class _FollowSpider(Spider): + name = "benchmark" + url: str + link_extractor = LinkExtractor() + + async def start(self) -> AsyncIterator[Any]: + yield Request(self.url, dont_filter=True) + + def parse(self, response: Response) -> Any: + yield _Page( + url=response.url, + anchors=response.css("a::text").getall(), + ) + for link in self.link_extractor.extract_links(response): # type: ignore[arg-type] + yield Request(link.url) + + +class _Pipeline: + def process_item(self, item: Any) -> Any: + return item + + +def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Per-request overhead of a crawl over HTTP. + + The pages are small on purpose, so that the cost of parsing them stays + negligible next to the cost of moving requests and responses through the + engine, the middlewares and the download handler. + """ + query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) + url = mockserver.url(f"/follow?{query}") + settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} + + def run() -> None: + crawler = crawl(_FollowSpider, settings, url=url) + assert crawler.stats + assert crawler.stats.get_value("item_scraped_count") == PAGES + 1 + + benchmark(run) diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 70c26e6d0..5306e3bf8 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase): _, _, stderr = self.crawl(code, proj_path, args=args) return stderr + def test_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("crawl", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + def test_multiple_spiders(self, proj_path: Path) -> None: + returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path) + assert returncode == 2 + assert ( + "running 'scrapy crawl' with more than one spider is not supported" in err + ) + def test_no_output(self, proj_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d98dac968..c6a3afc91 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -2,13 +2,24 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: + from pathlib import Path + from tests.mockserver.http import MockServer class TestFetchCommand: + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("fetch", *args) + assert returncode == 2 + assert "Usage" in out + def test_output(self, mockserver: MockServer) -> None: _, out, _ = proc("fetch", mockserver.url("/text")) assert out.strip() == "Works" @@ -36,3 +47,24 @@ class TestFetchCommand: "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" + + +class TestFetchCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + custom_settings = {"USER_AGENT": "myspider-user-agent"} +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, err = proc( + "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path + ) + assert "myspider-user-agent" in out, err diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 8bb6a2332..ddf25af4c 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase): assert call("genspider", "--dump=basic", cwd=proj_path) == 0 assert call("genspider", "-d", "basic", cwd=proj_path) == 0 + @pytest.mark.parametrize( + "args", + [("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")], + ) + def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, err = proc("genspider", *args, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find template: nonexistent" in out + assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists() + + def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None: + """The module name, unlike the spider name, is prefixed with a letter.""" + _, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path) + assert "Created spider '1st_spider'" in out, err + spider = proj_path / self.project_name / "spiders" / "a1st_spider.py" + assert spider.exists() + assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None + @pytest.mark.skipif( sys.platform == "win32", reason="requires a POSIX shell editor script" ) @@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase): ) def test_same_name_as_project(self, proj_path: Path) -> None: - assert call("genspider", self.project_name, cwd=proj_path) == 2 + _, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path) + assert "Cannot create a spider with the same name as your project" in out, err assert not ( proj_path / self.project_name / "spiders" / f"{self.project_name}.py" ).exists() diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 772cc82e2..e434055b7 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import re from typing import TYPE_CHECKING +from urllib.parse import urlparse import pytest @@ -552,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' assert file_path.read_text(encoding="utf-8") == content + @pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")]) + def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None: + returncode, out, _ = proc("parse", *args, cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + + @pytest.mark.parametrize( + ("option", "message"), + [ + ("--meta", "Invalid -m/--meta value"), + ("-m", "Invalid -m/--meta value"), + ("--cbkwargs", "Invalid --cbkwargs value"), + ], + ) + def test_invalid_json( + self, option: str, message: str, proj_path: Path, mockserver: MockServer + ) -> None: + returncode, _, err = proc( + "parse", + "--spider", + self.spider_name, + option, + "{invalid", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 2 + assert message in err + + def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None: + returncode, _, err = proc( + "parse", + "--spider", + "nonexistent", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 0, err + assert "Unable to find spider: nonexistent" in err + + def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None: + """Without --spider, the spider is chosen based on the URL.""" + url = mockserver.url("/html") + # The spider name doubles as a domain of the spider, and it is matched + # against the netloc of the URL, hence the port. + (proj_path / self.project_name / "spiders" / "urlspider.py").write_text( + f""" +import scrapy + +class UrlSpider(scrapy.Spider): + name = "{urlparse(url).netloc}" + + def parse(self, response): + return [{{"found_by_url": True}}] +""", + encoding="utf-8", + ) + returncode, out, err = proc("parse", url, cwd=proj_path) + assert returncode == 0, err + assert "Unable to find spider for" not in err + assert "{'found_by_url': True}" in out + + def test_legacy_item_processor( + self, proj_path: Path, mockserver: MockServer + ) -> None: + """--pipelines supports an ITEM_PROCESSOR without process_item_async().""" + (proj_path / self.project_name / "legacy.py").write_text( + """ +import logging + +from twisted.internet.defer import succeed + + +class LegacyItemProcessor: + @classmethod + def from_crawler(cls, crawler): + return cls() + + def open_spider(self, spider): + return succeed(None) + + def close_spider(self, spider): + return succeed(None) + + def process_item(self, item, spider): + logging.info("Legacy item processor!") + return succeed(item) +""", + encoding="utf-8", + ) + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "-s", + f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor", + mockserver.url("/html"), + cwd=proj_path, + ) + assert "INFO: Legacy item processor!" in stderr + + @pytest.mark.parametrize("verbose", [True, False]) + def test_no_items_no_links( + self, verbose: bool, proj_path: Path, mockserver: MockServer + ) -> None: + args = ["--verbose"] if verbose else [] + _, out, err = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "--noitems", + "--nolinks", + *args, + mockserver.url("/html"), + cwd=proj_path, + ) + assert "# Scraped Items" not in out, err + assert "# Requests" not in out + def test_parse_add_options(self): command = parse.Command() command.settings = Settings() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 2b410b5c6..11036eaeb 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -136,6 +136,12 @@ class MySpider(scrapy.Spider): log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log + @pytest.mark.parametrize("args", [(), ("a.py", "b.py")]) + def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("runspider", *args) + assert returncode == 2 + assert "Usage" in out + def test_runspider_file_not_found(self) -> None: _, _, log = proc("runspider", "some_non_existent_file") assert "File not found: some_non_existent_file" in log diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index f24200f53..29667a1ae 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test @@ -162,6 +163,33 @@ class TestShellCommand: assert ret == 0, out +class TestShellCommandWithSpider(TestProjectBase): + @pytest.fixture(autouse=True) + def create_files(self, proj_path: Path) -> None: + (proj_path / self.project_name / "spiders" / "myspider.py").write_text( + """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" +""", + encoding="utf-8", + ) + + def test_spider(self, proj_path: Path, mockserver: MockServer) -> None: + ret, out, err = proc( + "shell", + "--spider", + "myspider", + mockserver.url("/text"), + "-c", + "spider.name", + cwd=proj_path, + ) + assert ret == 0, err + assert out.strip() == "myspider" + + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: args = ( diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e687e811..f20ecc153 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,21 +3,27 @@ from __future__ import annotations import argparse import json import sys +from pathlib import Path from typing import TYPE_CHECKING import pytest import scrapy from scrapy.cmdline import _pop_command_name, execute -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path from tests.utils.bases.commands import TestProjectBase -from tests.utils.cmdline import call, proc, write_recording_editor +from tests.utils.cmdline import ( + call, + proc, + write_recording_browser, + write_recording_editor, +) if TYPE_CHECKING: - from pathlib import Path + from tests.mockserver.http import MockServer class EmptyCommand(ScrapyCommand): @@ -107,6 +113,93 @@ class TestCommandSettings: ) +class TestGlobalOptions: + """Tests for the options that every command supports.""" + + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = "myspider" + + async def start(self): + self.logger.debug("It works!") + return + yield +""" + + @pytest.fixture + def spider_path(self, tmp_path: Path) -> Path: + path = tmp_path / "myspider.py" + path.write_text(self.spider_code, encoding="utf-8") + return path + + def test_invalid_set(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO") + assert returncode == 2 + assert "Invalid -s value, use -s NAME=VALUE" in err + + def test_invalid_spider_argument(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO") + assert returncode == 2 + assert "Invalid -a value, use -a NAME=VALUE" in err + + def test_logfile(self, tmp_path: Path, spider_path: Path) -> None: + logfile = tmp_path / "scrapy.log" + returncode, _, err = proc( + "runspider", str(spider_path), "--logfile", str(logfile) + ) + assert returncode == 0, err + assert "It works!" in logfile.read_text(encoding="utf-8") + assert "It works!" not in err + + def test_loglevel(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO") + assert returncode == 0, err + assert "It works!" not in err + assert "Spider closed (finished)" in err + + def test_nolog(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--nolog") + assert returncode == 0, err + assert not err + + def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None: + pidfile = tmp_path / "scrapy.pid" + returncode, _, err = proc( + "runspider", str(spider_path), "--pidfile", str(pidfile) + ) + assert returncode == 0, err + assert pidfile.read_text(encoding="utf-8").strip().isdigit() + + def test_pdb(self, spider_path: Path) -> None: + returncode, _, err = proc("runspider", str(spider_path), "--pdb") + assert returncode == 0, err + assert "It works!" in err + + +class TestSettingsCommand: + @pytest.mark.parametrize( + ("option", "setting", "expected"), + [ + ("--get", "BOT_NAME", "scrapybot"), + ("--getbool", "COOKIES_ENABLED", "True"), + ("--getint", "CONCURRENT_REQUESTS", "16"), + ("--getfloat", "DOWNLOAD_DELAY", "0.0"), + ("--getlist", "SPIDER_MODULES", "[]"), + ], + ) + def test_get(self, option: str, setting: str, expected: str) -> None: + returncode, out, err = proc("settings", option, setting) + assert returncode == 0, err + assert out.startswith(expected) + + def test_no_option(self) -> None: + returncode, out, err = proc("settings") + assert returncode == 0, err + assert not out + + class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed.""" @@ -577,18 +670,31 @@ class TestBenchCommand: class TestViewCommand: - def test_methods(self) -> None: - command = view.Command() - command.settings = Settings() - parser = argparse.ArgumentParser( - prog="scrapy", - prefix_chars="-", - formatter_class=ScrapyHelpFormatter, - conflict_handler="resolve", + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX shell browser script" + ) + def test_view( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer + ) -> None: + opened = tmp_path / "opened.txt" + browser = tmp_path / "fake-browser.sh" + write_recording_browser(browser, opened) + monkeypatch.setenv("BROWSER", str(browser)) + + returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path) + + assert returncode == 0, err + url = opened.read_text(encoding="utf-8") + assert url.startswith("file://") + body = Path(url.removeprefix("file://")).read_text(encoding="utf-8") + assert "

Works

" in body + + def test_non_text_response(self, mockserver: MockServer) -> None: + returncode, _, err = proc( + "view", mockserver.url("/static/files/images/scrapy.png") ) - command.add_options(parser) - assert command.short_desc() == "Open URL in browser, as seen by Scrapy" - assert "URL using the Scrapy downloader and show its" in command.long_desc() + assert returncode == 0, err + assert "Cannot view a non-text response." in err class TestEditCommand(TestProjectBase): @@ -615,6 +721,11 @@ class TestEditCommand(TestProjectBase): assert returncode == 1 assert "Spider not found: nonexistent" in err + def test_edit_no_spider(self, proj_path: Path) -> None: + returncode, out, _ = proc("edit", cwd=proj_path) + assert returncode == 2 + assert "Usage" in out + class TestHelpMessage(TestProjectBase): @pytest.mark.parametrize( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..3e4139b3e 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,6 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from scrapy import Request from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): await self._assert_factory_works(server_url, client_context_factory) +@pytest.mark.parametrize( + ("concurrency", "active", "expected"), + [ + (2, 1, False), + (2, 2, True), + (0, 0, False), + (0, 2, False), + ], +) +def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + downloader = Downloader(crawler) + downloader.active = {Request(f"https://example.com/{i}") for i in range(active)} + assert downloader.needs_backout() is expected + downloader.close() + + @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index fe2f83161..733b6797d 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,8 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from tests.utils import async_sleep, get_script_run_env +from scrapy.utils.asyncio import sleep +from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - await async_sleep(0.01) + await sleep(0.01) p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 5ceb93382..976daacaf 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): @pytest.mark.requires_internet class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass + + +@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) +@coroutine_test +async def test_pool_limits(concurrency: int, expected: int | None) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + handler = build_from_crawler(HttpxDownloadHandler, crawler) + try: + assert handler._limits.max_connections == expected + assert handler._limits.max_keepalive_connections == expected + finally: + await handler.close() diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ce56ee11d..6e8486eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,6 +14,7 @@ import pytest from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -24,6 +25,14 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +class AlwaysStalePolicy(DummyPolicy): + """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always + revalidates cached responses.""" + + def is_cached_response_fresh(self, cachedresponse, request): + return False + + class TestBase: """Base class with common setup and helper methods.""" @@ -282,6 +291,21 @@ class DummyPolicyTestMixin(PolicyTestMixin): self.assertEqualResponse(self.response, response) assert "cached" in response.flags + def test_revalidation_keeps_cached_response(self): + # The dummy policy considers every cached response valid, so a policy + # that subclasses it to force revalidation always gets the cached + # response back, whatever the new response is. + with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw: + assert mw.process_request(self.request) is None + mw.process_response(self.request, self.response) + + assert mw.process_request(self.request) is None + fresh_response = self.response.replace(body=b"new body") + response = mw.process_response(self.request, fresh_response) + self.assertEqualResponse(self.response, response) + assert "cached" in response.flags + assert mw.stats.get_value("httpcache/revalidate") == 1 + class RFC2616PolicyTestMixin(PolicyTestMixin): """Mixin containing RFC2616 policy specific test methods.""" @@ -553,6 +577,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): else: assert "cached" in res5.flags + def test_middleware_ignore_schemes(self): + # file responses are not cached by default + req = Request("file:///tmp/t.txt") + res = Response(req.url, headers={"Expires": self.tomorrow}) + with self._middleware() as mw: + assert mw.process_request(req) is None + mw.process_response(req, res) + + assert mw.storage.retrieve_response(mw.crawler.spider, req) is None + assert mw.process_request(req) is None + + def test_max_stale_with_value(self): + # A response that expired one day ago. + headers = {"Date": self.yesterday, "Expires": self.yesterday} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + self._process_requestresponse(mw, req0, res0) + + # max-stale greater than the staleness of the cached response + req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"}) + res1 = mw.process_request(req1) + assert isinstance(res1, Response) + assert "cached" in res1.flags + + # max-stale lower than the staleness of the cached response + req2 = req0.replace(headers={"Cache-Control": "max-stale=60"}) + assert mw.process_request(req2) is None + + # a non-integer max-stale value is ignored + req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"}) + assert mw.process_request(req3) is None + + def test_response_dated_in_the_future(self): + # A Date header ahead of the local clock must not make the cached + # response look aged. + headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"} + with self._middleware() as mw: + req0 = Request("http://example.com") + res0 = Response(req0.url, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + assert "cached" not in res1.flags + + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert "cached" in res2.flags + def test_process_exception(self): with self._middleware() as mw: res0 = Response(self.request.url, headers={"Expires": self.yesterday}) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f82041a62..1f2575f8f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -74,6 +76,21 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c15c396d3..14ec3d184 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.asyncio import call_later +from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -65,23 +64,23 @@ class TestMain: async def start(self): yield Request("data:,a") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await async_sleep(seconds) + await sleep(seconds) yield Request("data:,c") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 3b997767e..b857728ba 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC): ie = self._get_exporter(fields_to_export={"name": "名稱"}) assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")] + def test_field_order(self): + item = self.item_class(age="22", name="John\xa3") + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"] + + def test_field_order_dict_item(self): + ie = self._get_exporter() + assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"] + assert [ + name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"}) + ] == ["age", "name"] + def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py index a474725d8..76e8ca5d6 100644 --- a/tests/test_extension_memusage.py +++ b/tests/test_extension_memusage.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import sys +from typing import TYPE_CHECKING import pytest @@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.utils import OneShotLoop +from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + # MemoryUsage relies on the stdlib 'resource' module (not available on Windows) pytestmark = pytest.mark.skipif( sys.platform.startswith("win"), @@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif( MB = 1024 * 1024 +class TwoShotLoop(OneShotLoop): + """Like :class:`OneShotLoop`, but runs the check twice.""" + + def start(self, interval: float, now: bool = True) -> None: + super().start(interval, now=now) + self.func() + + class _LoopSpider(Spider): name = "loop-data-spider" @@ -50,6 +63,49 @@ def test_memusage_disabled() -> None: MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) +def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None: + # The Scrapy shell starts the engine without opening a spider, so the + # whole crawler is stopped instead of a spider being closed. + _, out, err = proc( + "shell", + mockserver.url("/text"), + "-c", + "response.status", + "--set", + "MEMUSAGE_LIMIT_MB=1", + ) + assert "Memory usage exceeded 1MiB" in err + assert "200" in out + + +@coroutine_test +async def test_memusage_below_thresholds_logs_peak( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 100, + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=1) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") is None + assert crawler.stats.get_value("memusage/warning_reached") is None + assert crawler.stats.get_value("memusage/max") == 25 * MB + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records) + + @coroutine_test async def test_memusage_limit_closes_spider_with_reason_and_error_log( caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch @@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish( "LOG_LEVEL": "INFO", } - # Avoid background LoopingCall that can log after the test finishes. - monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid background LoopingCall that can log after the test finishes; check + # twice, since the warning is only meant to be reported once. + monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop) monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) @@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish( assert crawler.stats assert crawler.stats.get_value("memusage/warning_reached") == 1 assert crawler.stats.get_value("finish_reason") == "finished" - assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) + warnings_logged = [ + r for r in caplog.records if "memory usage reached" in r.getMessage().lower() + ] + assert len(warnings_logged) == 1 diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ffe7a0dc7..723fb6e17 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,13 @@ from __future__ import annotations import datetime +import json +import logging from typing import TYPE_CHECKING, Any +import pytest + +from scrapy.exceptions import NotConfigured from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler @@ -86,6 +91,14 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + def test_no_interval(self): + with pytest.raises(NotConfigured): + extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0}) + + def test_nothing_enabled(self): + with pytest.raises(NotConfigured): + extension({"LOGSTATS_INTERVAL": 60}) + @coroutine_test async def test_log_delta(self): def emulate( @@ -212,3 +225,26 @@ class TestPeriodicLog: {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + + @coroutine_test + async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None: + settings = { + "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "LOGSTATS_INTERVAL": 30, + } + crawler = get_crawler(MetaSpider, settings) + with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"): + await crawler.crawl_async() + + records = [ + r for r in caplog.records if r.name == "scrapy.extensions.periodic_log" + ] + assert records, "PeriodicLog logged nothing" + # Only the timing section is enabled, and it is logged on spider close. + data = json.loads(records[-1].getMessage()) + assert list(data) == ["time"] + assert data["time"]["log_interval"] == 30 + assert data["time"]["log_interval_real"] >= 0 + assert data["time"]["elapsed"] >= 0 + assert data["time"]["start_time"] <= data["time"]["utcnow"] diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 20c801558..fca0e3153 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -7,7 +7,8 @@ import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from scrapy.extensions.telnet import TelnetConsole +from scrapy import Spider +from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test @@ -16,16 +17,20 @@ if TYPE_CHECKING: from collections.abc import Generator from scrapy.crawler import Crawler + from scrapy.http import Response pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor -def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler: +def _get_crawler( + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, +) -> Crawler: settings = { "TELNETCONSOLE_ENABLED": True, **(settings_dict or {}), } - return get_crawler(settings_dict=settings) + return get_crawler(spidercls, settings_dict=settings) @contextmanager @@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None: console = TelnetConsole(_get_crawler(settings_dict=settings)) with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): console.start_listening() + + +@coroutine_test +async def test_telnet_vars() -> None: + """Log into the console of a running crawl, which is when the telnet + variables are built.""" + received: list[dict[str, Any]] = [] + + def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None: + received.append(telnet_vars) + + class TelnetSpider(Spider): + name = "telnet" + start_urls = ["data:,"] + + async def parse(self, response: Response) -> None: + assert self.crawler.extensions + console = next( + ext + for ext in self.crawler.extensions.middlewares + if isinstance(ext, TelnetConsole) + ) + creds = credentials.UsernamePassword( + console.username.encode("utf8"), console.password.encode("utf8") + ) + portal = console.protocol().protocolArgs[0] + await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol)) + + crawler = _get_crawler(TelnetSpider) + crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars) + await crawler.crawl_async() + + assert len(received) == 1 + telnet_vars = received[0] + assert telnet_vars["crawler"] is crawler + assert telnet_vars["engine"] is crawler.engine + assert telnet_vars["spider"] is crawler.spider + assert telnet_vars["extensions"] is crawler.extensions + assert telnet_vars["stats"] is crawler.stats + assert telnet_vars["settings"] is crawler.settings + assert callable(telnet_vars["est"]) + assert callable(telnet_vars["p"]) + assert callable(telnet_vars["prefs"]) + assert "telnetconsole.html" in telnet_vars["help"] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 92c414bef..40a763efd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from scrapy.extensions.feedexport import ( FeedExporter, FeedSlot, FileFeedStorage, + ItemFilter, apply_uri_params, ) from scrapy.utils.python import to_unicode @@ -675,14 +676,14 @@ class TestFeedExport(TestFeedExportBase): formats = { "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", - "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', + "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n' ), "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\nworld3" + b"bar1spam1\n" + b"bar2world2\nworld3" b"spam3\n" ), } @@ -740,8 +741,8 @@ class TestFeedExport(TestFeedExportBase): "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', "xml": ( b'\n\n' - b"bar1spam1\n" - b"world2bar2\n" + b"bar1spam1\n" + b"bar2world2\n" ), "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n', } @@ -1289,6 +1290,13 @@ class TestFeedExporterSignals: assert self.feed_exporter_closed_received +class TestItemFilter: + def test_no_feed_options(self): + item_filter = ItemFilter(None) + assert item_filter.item_classes == () + assert item_filter.accepts(MyItem({"foo": "bar"})) + + class TestFeedExportInit: def test_unsupported_storage(self): settings = { @@ -1300,6 +1308,24 @@ class TestFeedExportInit: with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_disabled_storage(self, caplog: pytest.LogCaptureFixture): + class DisabledFeedStorage: + def __init__(self, uri, *, feed_options=None): + raise NotConfigured("not today") + + settings = { + "FEED_STORAGES": {"disabled": DisabledFeedStorage}, + "FEEDS": { + "disabled://uri": {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured): + FeedExporter.from_crawler(crawler) + assert ( + "Disabled feed storage scheme: disabled. Reason: not today" in caplog.text + ) + def test_unsupported_format(self): settings = { "FEEDS": { diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index f120ce36f..36d8586ce 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any import pytest +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.test import get_crawler from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test @@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() + def test_tell_reports_target_file_position(self): + """Exporters that wrap the file they get, e.g. through + :class:`io.TextIOWrapper`, need it to report a position.""" + file = BytesIO() + manager = PostProcessingManager([self.MyPlugin1], file, {}) + assert manager.tell() == 0 + manager.write(b"foo") + assert manager.tell() == file.tell() == 3 + @coroutine_test async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 66488540f..4d28872b7 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import os import string +import sys import tempfile from io import BytesIO from pathlib import Path @@ -14,6 +15,7 @@ import pytest from w3lib.url import path_to_file_uri import scrapy +from scrapy.exceptions import NotConfigured from scrapy.extensions.feedexport import ( BlockingFeedStorage, FileFeedStorage, @@ -166,6 +168,12 @@ class TestFTPFeedStorage: st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) assert st.password == string.punctuation + def test_uri_without_hostname(self): + with pytest.raises( + ValueError, match="Got a storage URI without a hostname: ftp:///some_path" + ): + FTPFeedStorage("ftp:///some_path") + class MyBlockingFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: @@ -205,6 +213,13 @@ class TestBlockingFeedStorage: b.open(spider=spider) +def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boto3", None) + monkeypatch.setitem(sys.modules, "boto3.session", None) + with pytest.raises(NotConfigured, match="missing boto3 library"): + S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + + @pytest.mark.requires_boto3 class TestS3FeedStorage: def test_parse_credentials(self): @@ -381,6 +396,41 @@ class TestS3FeedStorage: assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name + def test_init_without_max_pool_connections(self) -> None: + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.max_pool_connections is None + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 10 + + def test_init_with_max_pool_connections(self) -> None: + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + max_pool_connections=30, + ) + assert storage.max_pool_connections == 30 + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == 30 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_from_crawler_max_pool_connections( + self, settings: dict[str, Any], expected: int + ) -> None: + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.max_pool_connections == expected + config: Any = storage.s3_client.meta.config + assert config.max_pool_connections == expected + @coroutine_test async def test_store_without_acl(self): storage = S3FeedStorage( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 97daa514e..4e7fb118b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -895,6 +895,33 @@ class TestS3FilesStore: stub.assert_no_pending_responses() + def test_default_max_pool_connections(self) -> None: + store = S3FilesStore("s3://mybucket/prefix/") + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == 10 + + @pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, 10), + ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20), + ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30), + ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30), + ], + ) + def test_max_pool_connections( + self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int + ) -> None: + # restores the value that FilesPipeline.from_crawler() sets on the class + monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None) + crawler = get_crawler( + settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings} + ) + store = FilesPipeline.from_crawler(crawler).store + assert isinstance(store, S3FilesStore) + config: Any = store.s3_client.meta.config + assert config.max_pool_connections == expected + class TestGCSFilesStore: @staticmethod diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index da94a4e95..ea67877f8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -4,6 +4,7 @@ from scrapy.robotstxt import ( ProtegoRobotParser, PythonRobotParser, RerpRobotParser, + RobotParser, decode_robotstxt, ) from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER @@ -78,6 +79,16 @@ class BaseRobotParserTest: assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -102,6 +113,22 @@ class BaseRobotParserTest: assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") +class TestRobotParser: + def test_crawl_delay_unsupported(self): + class AllowAllRobotParser(RobotParser): + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + return cls() + + def allowed(self, url, user_agent): + return True + + rp = AllowAllRobotParser.from_crawler( + crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n" + ) + assert rp.crawl_delay("*") is None + + class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 9b7eb22fa..4bd54acd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, + call_later, is_asyncio_available, + sleep, ) from tests.utils.decorators import coroutine_test @@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None: assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_sleep() -> None: + events: list[str] = [] + call_later(0.05, events.append, "call_later") + await sleep(0.1) + events.append("sleep") + assert events == ["call_later", "sleep"] + + @pytest.mark.only_asyncio class TestParallelAsyncio: """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ad9aa3dff..0dea8af6d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,10 +1,38 @@ from __future__ import annotations +import subprocess +import sys from importlib.util import find_spec +from io import BytesIO +from typing import TYPE_CHECKING import pytest +from pexpect import EOF -from scrapy.utils.console import get_shell_embed_func +from scrapy.utils.console import get_shell_embed_func, start_python_console +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + from pathlib import Path + +CONSOLE = """ +from scrapy.utils.console import start_python_console + +start_python_console(banner="SHELL-READY", shells=["ipython"]) +""" + +CONSOLE_IN_RUNNING_LOOP = """ +import asyncio + +from scrapy.utils.console import start_python_console + + +async def main(): + start_python_console(banner="SHELL-READY", shells=["ipython"]) + + +asyncio.run(main()) +""" def test_get_shell_embed_func(): @@ -59,3 +87,90 @@ def test_get_shell_embed_func_default(): else: expected = "_embed_standard_shell" assert shell.__name__ == expected + + +@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed") +class TestIPythonShell: + """Starting an IPython shell, with and without an asyncio event loop already + running in the calling thread. The latter happens when inspect_response() is + called from a spider callback while using the asyncio reactor.""" + + @staticmethod + def _env(tmp_path: Path) -> dict[str, str]: + env = get_testenv() + # Keep IPython away from the profile and history of the user running the tests. + env["IPYTHONDIR"] = str(tmp_path) + return env + + def test_simple_prompt(self, tmp_path: Path) -> None: + """IPython falls back to its simple prompt, which needs no event loop, + when stdin is not a TTY.""" + env = self._env(tmp_path) + p = subprocess.run( + [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP], + check=False, + capture_output=True, + encoding="utf-8", + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + output = p.stdout + p.stderr + assert "SHELL-READY" in output + assert p.returncode == 0, output + + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX pseudo-terminal" + ) + @pytest.mark.parametrize( + "script", + [CONSOLE, CONSOLE_IN_RUNNING_LOOP], + ids=["no_running_loop", "running_loop"], + ) + def test_tty(self, tmp_path: Path, script: str) -> None: + """IPython uses prompt_toolkit, which needs an event loop of its own, + when stdin is a TTY.""" + # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX. + from pexpect import spawn # noqa: PLC0415 + + env = self._env(tmp_path) + env.pop("IPY_TEST_SIMPLE_PROMPT", None) + env["TERM"] = "xterm" + logfile = BytesIO() + p = spawn( + sys.executable, + ["-c", script], + env=env, + timeout=60, + ) + p.logfile_read = logfile + try: + # Wait for the prompt, which prompt_toolkit draws once it is done + # querying the terminal, before typing into it. + p.expect(r"In \[") + p.sendline("21*2") + p.expect_exact("42") + p.sendline("exit()") + p.expect(EOF) + finally: + p.close() + output = logfile.getvalue().decode() + assert "Traceback" not in output + assert p.exitstatus == 0, output + + +def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: + def embed(namespace: dict[str, object], banner: str) -> None: + raise SystemExit + + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: embed + ) + start_python_console() + + +def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "scrapy.utils.console.get_shell_embed_func", lambda shells: None + ) + start_python_console() diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py index 807294a57..4c29d2917 100644 --- a/tests/test_utils_decorators.py +++ b/tests/test_utils_decorators.py @@ -1,7 +1,8 @@ from __future__ import annotations +import sys import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import pytest from twisted.internet.defer import Deferred @@ -12,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from tests.utils.decorators import coroutine_test if TYPE_CHECKING: - from collections.abc import AsyncGenerator + from collections.abc import AsyncGenerator, Callable class TestDeprecated: @@ -77,6 +78,31 @@ class TestWarnSpiderArg: ): assert parse("response", spider="spider") == "response" + @pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", + ) + def test_sync_warns_with_unresolvable_annotations(self): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def parse(response: OnlyAtTypeCheckingTime," + " spider: OnlyAtTypeCheckingTime | None = None): return response", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + parse_func: Callable[..., str] = namespace["parse"] + parse = _warn_spider_arg(parse_func) + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + def test_sync_no_warning_without_spider_arg(self): @_warn_spider_arg def parse(response: str, spider: str | None = None) -> str: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c3b5dfc99..099b5ccc2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,7 +4,7 @@ import functools import operator import platform import sys -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar import pytest @@ -190,6 +190,25 @@ def test_get_func_args(): ] +@pytest.mark.skipif( + sys.version_info < (3, 14), + reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)", +) +def test_get_func_args_unresolvable_annotations(): + # dont_inherit=True, or the module's future import stringizes the annotations + namespace: dict[str, Any] = {} + exec( # pylint: disable=exec-used + compile( + "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass", + "", + "exec", + dont_inherit=True, + ), + namespace, + ) + assert get_func_args(namespace["f"]) == ["a", "b"] + + @pytest.mark.parametrize( ("value", "expected"), [ diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index b0632a7ea..b27c5ade7 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import os from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +7,6 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.defer import maybe_deferred_to_future if TYPE_CHECKING: from collections.abc import Callable @@ -23,13 +20,6 @@ def twisted_sleep(seconds: float): return d -async def async_sleep(seconds: float) -> None: - if is_asyncio_available(): - await asyncio.sleep(seconds) - else: - await maybe_deferred_to_future(twisted_sleep(seconds)) - - def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 62dff3d4c..095cb17a7 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -46,3 +46,16 @@ def write_recording_editor(editor: Path) -> None: open (its last argument) into the file given as its first argument.""" editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") editor.chmod(0o755) + + +def write_recording_browser(browser: Path, recorded: Path) -> None: + """Create an executable browser script that writes the URL it is asked to + open into *recorded*. + + ``webbrowser`` only passes the URL to the command from the ``BROWSER`` + environment variable, hence the hardcoded output path. + """ + browser.write_text( + f'#!/bin/sh\nprintf "%s" "$1" > "{recorded}"\n', encoding="utf-8" + ) + browser.chmod(0o755) diff --git a/tox.ini b/tox.ini index e10a7cc0c..8331e2ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 envlist = pre-commit pylint @@ -30,6 +30,7 @@ envlist = botocore pypy3 pypy3-extra-deps + benchmark minversion = 1.7.0 [test-requirements] @@ -189,8 +190,8 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 - ipython==7.1.0 - ptpython==2.0.1 + ipython==8.15.0 + ptpython==3.0.23 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" @@ -317,3 +318,20 @@ setenv = {[min]setenv} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore + + +# CPU benchmarks, tracked on CodSpeed. +# +# pytest-twisted is left out on purpose: benchmarked code must be callable +# synchronously, so tests/benchmarks drives the reactor itself. + +[testenv:benchmark] +basepython = python3.14 +deps = + pytest >= 8.4.1 + pytest-codspeed +passenv = + *codspeed* + *ci* +commands = + pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation