Merge branch 'master' into http2

conftest.py

@@ -2,6 +2,8 @@ from pathlib import Path
 
 import pytest
 
+from scrapy.utils.reactor import install_reactor
+
 from tests.keys import generate_keys
 
 
@@ -40,6 +42,14 @@ def pytest_collection_modifyitems(session, config, items):
         pass
 
 
+def pytest_addoption(parser):
+    parser.addoption(
+        "--reactor",
+        default="default",
+        choices=["default", "asyncio"],
+    )
+
+
 @pytest.fixture(scope='class')
 def reactor_pytest(request):
     if not request.cls:
@@ -55,5 +65,10 @@ def only_asyncio(request, reactor_pytest):
         pytest.skip('This test is only run with --reactor=asyncio')
 
 
+def pytest_configure(config):
+    if config.getoption("--reactor") == "asyncio":
+        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+
 # Generate localhost certificate files, needed by some tests
 generate_keys()

docs/faq.rst

@@ -145,6 +145,41 @@ How can I make Scrapy consume less memory?
 
 See previous question.
 
+How can I prevent memory errors due to many allowed domains?
+------------------------------------------------------------
+
+If you have a spider with a long list of
+:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider
+replacing the default
+:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware
+with a :ref:`custom spider middleware <custom-spider-middleware>` that requires
+less memory. For example:
+
+-   If your domain names are similar enough, use your own regular expression
+    instead joining the strings in
+    :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular
+    expression.
+
+-   If you can `meet the installation requirements`_, use pyre2_ instead of
+    Python’s re_ to compile your URL-filtering regular expression. See
+    :issue:`1908`.
+
+See also other suggestions at `StackOverflow`_.
+
+.. note:: Remember to disable
+   :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable
+   your custom implementation::
+
+       SPIDER_MIDDLEWARES = {
+           'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
+           'myproject.middlewares.CustomOffsiteMiddleware': 500,
+       }
+
+.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation
+.. _pyre2: https://github.com/andreasvc/pyre2
+.. _re: https://docs.python.org/library/re.html
+.. _StackOverflow: https://stackoverflow.com/q/36440681/939364
+
 Can I use Basic HTTP Authentication in my spiders?
 --------------------------------------------------
 

docs/intro/tutorial.rst

@@ -464,7 +464,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports
 
     scrapy crawl quotes -O quotes.json
 
-That will generate an ``quotes.json`` file containing all scraped items,
+That will generate a ``quotes.json`` file containing all scraped items,
 serialized in `JSON`_.
 
 The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead

docs/topics/exporters.rst

@@ -50,18 +50,19 @@ value of one of their fields::
             self.year_to_exporter = {}
 
         def close_spider(self, spider):
-            for exporter in self.year_to_exporter.values():
+            for exporter, xml_file in self.year_to_exporter.values():
                 exporter.finish_exporting()
+                xml_file.close()
 
         def _exporter_for_item(self, item):
             adapter = ItemAdapter(item)
             year = adapter['year']
             if year not in self.year_to_exporter:
-                f = open(f'{year}.xml', 'wb')
-                exporter = XmlItemExporter(f)
+                xml_file = open(f'{year}.xml', 'wb')
+                exporter = XmlItemExporter(xml_file)
                 exporter.start_exporting()
-                self.year_to_exporter[year] = exporter
-            return self.year_to_exporter[year]
+                self.year_to_exporter[year] = (exporter, xml_file)
+            return self.year_to_exporter[year][0]
 
         def process_item(self, item, spider):
             exporter = self._exporter_for_item(item)

pytest.ini

@@ -18,7 +18,6 @@ addopts =
     --ignore=docs/topics/stats.rst
     --ignore=docs/topics/telnetconsole.rst
     --ignore=docs/utils
-twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
 flake8-max-line-length = 119

scrapy/utils/curl.py

@@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments:
     curl_parser.add_argument(*argument, action='store_true')
 
 
+def _parse_headers_and_cookies(parsed_args):
+    headers = []
+    cookies = {}
+    for header in parsed_args.headers or ():
+        name, val = header.split(':', 1)
+        name = name.strip()
+        val = val.strip()
+        if name.title() == 'Cookie':
+            for name, morsel in SimpleCookie(val).items():
+                cookies[name] = morsel.value
+        else:
+            headers.append((name, val))
+
+    if parsed_args.auth:
+        user, password = parsed_args.auth.split(':', 1)
+        headers.append(('Authorization', basic_auth_header(user, password)))
+
+    return headers, cookies
+
+
 def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
     """Convert a cURL command syntax to Request kwargs.
 
@@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
 
     result = {'method': method.upper(), 'url': url}
 
-    headers = []
-    cookies = {}
-    for header in parsed_args.headers or ():
-        name, val = header.split(':', 1)
-        name = name.strip()
-        val = val.strip()
-        if name.title() == 'Cookie':
-            for name, morsel in SimpleCookie(val).items():
-                cookies[name] = morsel.value
-        else:
-            headers.append((name, val))
-
-    if parsed_args.auth:
-        user, password = parsed_args.auth.split(':', 1)
-        headers.append(('Authorization', basic_auth_header(user, password)))
+    headers, cookies = _parse_headers_and_cookies(parsed_args)
 
     if headers:
         result['headers'] = headers

tests/requirements-py3.txt

@@ -2,10 +2,8 @@
 attrs
 dataclasses; python_version == '3.6'
 pyftpdlib
-# https://github.com/pytest-dev/pytest-twisted/issues/93
-pytest != 5.4, != 5.4.1
+pytest
 pytest-cov
-pytest-twisted >= 1.11
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures

tests/test_utils_defer.py

@@ -1,8 +1,10 @@
+from pytest import mark
 from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
 from scrapy.utils.defer import (
+    deferred_f_from_coro_f,
     iter_errback,
     mustbe_deferred,
     process_chain,
@@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase):
         self.assertEqual(out, [0, 1, 2, 3, 4])
         self.assertEqual(len(errors), 1)
         self.assertIsInstance(errors[0].value, ZeroDivisionError)
+
+
+class AsyncDefTestsuiteTest(unittest.TestCase):
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f(self):
+        pass
+
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f_generator(self):
+        yield
+
+    @mark.xfail(reason="Checks that the test is actually executed", strict=True)
+    @deferred_f_from_coro_f
+    async def test_deferred_f_from_coro_f_xfail(self):
+        raise Exception("This is expected to be raised")

tox.ini

@@ -17,7 +17,8 @@ deps =
     # Extras
     botocore>=1.4.87
     Pillow>=4.0.0
-    Twisted[http2]>=17.9.0
+    # Twisted 21+ causes issues in tests that use skipIf
+    Twisted[http2]>=17.9.0,<21
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID