Merge branch 'master' into response_ip_address

docs/intro/tutorial.rst

@@ -652,7 +652,7 @@ this time for scraping author information::
 
         def parse(self, response):
             author_page_links = response.css('.author + a')
-            yield from response.follow_all(author_links, self.parse_author)
+            yield from response.follow_all(author_page_links, self.parse_author)
 
             pagination_links = response.css('li.next a')
             yield from response.follow_all(pagination_links, self.parse)

docs/topics/downloader-middleware.rst

@@ -199,7 +199,7 @@ CookiesMiddleware
 
    This middleware enables working with sites that require cookies, such as
    those that use sessions. It keeps track of cookies sent by web servers, and
-   send them back on subsequent requests (from that spider), just like web
+   sends them back on subsequent requests (from that spider), just like web
    browsers do.
 
 The following settings can be used to configure the cookie middleware:
@@ -672,7 +672,7 @@ sometimes a more nuanced policy is desirable.
 
 This setting still respects ``Cache-Control: no-store`` directives in responses.
 If you don't want that, filter ``no-store`` out of the Cache-Control headers in
-responses you feedto the cache middleware.
+responses you feed to the cache middleware.
 
 .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
 
@@ -686,7 +686,7 @@ Default: ``[]``
 List of Cache-Control directives in responses to be ignored.
 
 Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
-upset at the traffic a spider can generate if it respects those
+upset at the traffic a spider can generate if it actually respects those
 directives. This allows to selectively ignore Cache-Control directives
 that are known to be unimportant for the sites being crawled.
 

scrapy/downloadermiddlewares/httpcompression.py

@@ -26,7 +26,7 @@ class HttpCompressionMiddleware(object):
 
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding',
-                                   b",".join(ACCEPTED_ENCODINGS))
+                                   b", ".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 

scrapy/downloadermiddlewares/retry.py

@@ -84,6 +84,6 @@ class RetryMiddleware(object):
             return retryreq
         else:
             stats.inc_value('retry/max_reached')
-            logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
+            logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})

scrapy/utils/spider.py

@@ -2,14 +2,15 @@ import logging
 import inspect
 
 from scrapy.spiders import Spider
-from scrapy.utils.misc import  arg_to_iter
+from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.misc import arg_to_iter
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
-    return arg_to_iter(result)
+    return arg_to_iter(deferred_from_coro(result))
 
 
 def iter_spider_classes(module):

tests/spiders.py

@@ -1,14 +1,18 @@
 """
 Some spiders used for testing and benchmarking
 """
+import asyncio
 import time
 from urllib.parse import urlencode
 
+from twisted.internet import defer
+
 from scrapy.http import Request
 from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.utils.test import get_from_asyncio_queue
 
 
 class MockServerSpider(Spider):
@@ -83,6 +87,36 @@ class SimpleSpider(MetaSpider):
         self.logger.info("Got response %d" % response.status)
 
 
+class AsyncDefSpider(SimpleSpider):
+
+    name = 'asyncdef'
+
+    async def parse(self, response):
+        await defer.succeed(42)
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+
+
+class AsyncDefAsyncioReturnSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_return'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        status = await get_from_asyncio_queue(response.status)
+        self.logger.info("Got response %d" % status)
+        return [{'id': 1}, {'id': 2}]
+
+
 class ItemSpider(FollowAllSpider):
 
     name = 'item'

tests/test_crawl.py

@@ -4,16 +4,19 @@ from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
 
+from pytest import mark
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
+from scrapy import signals
 from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
-                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
+                           SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
+                           AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
 
 
 class CrawlTestCase(TestCase):
@@ -312,6 +315,38 @@ with multiples lines
         self.assertIn("[errback] status 404", str(log))
         self.assertIn("[errback] status 500", str(log))
 
+    @defer.inlineCallbacks
+    def test_async_def_parse(self):
+        self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield self.runner.join()
+        self.assertIn("Got response 200", str(log))
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse(self):
+        runner = CrawlerRunner({"ASYNCIO_REACTOR": True})
+        runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        with LogCapture() as log:
+            yield runner.join()
+        self.assertIn("Got response 200", str(log))
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncio_parse_list(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        self.assertIn({'id': 1}, items)
+        self.assertIn({'id': 2}, items)
+
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
         crawler = self.runner.create_crawler(SingleRequestSpider)

tests/test_downloadermiddleware_httpcompression.py

@@ -48,7 +48,7 @@ class HttpCompressionTest(TestCase):
                 }
 
         response = Response('http://scrapytest.org/', body=body, headers=headers)
-        response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip,deflate'})
+        response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'})
         return response
 
     def test_process_request(self):
@@ -56,7 +56,7 @@ class HttpCompressionTest(TestCase):
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
         self.assertEqual(request.headers.get('Accept-Encoding'),
-                         b','.join(ACCEPTED_ENCODINGS))
+                         b', '.join(ACCEPTED_ENCODINGS))
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')