diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c9d00eb74..ee10048b5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -652,7 +652,7 @@ this time for scraping author information:: def parse(self, response): author_page_links = response.css('.author + a') - yield from response.follow_all(author_links, self.parse_author) + yield from response.follow_all(author_page_links, self.parse_author) pagination_links = response.css('li.next a') yield from response.follow_all(pagination_links, self.parse) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ae6d41809..8a760e53b 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -199,7 +199,7 @@ CookiesMiddleware This middleware enables working with sites that require cookies, such as those that use sessions. It keeps track of cookies sent by web servers, and - send them back on subsequent requests (from that spider), just like web + sends them back on subsequent requests (from that spider), just like web browsers do. The following settings can be used to configure the cookie middleware: @@ -672,7 +672,7 @@ sometimes a more nuanced policy is desirable. This setting still respects ``Cache-Control: no-store`` directives in responses. If you don't want that, filter ``no-store`` out of the Cache-Control headers in -responses you feedto the cache middleware. +responses you feed to the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -686,7 +686,7 @@ Default: ``[]`` List of Cache-Control directives in responses to be ignored. Sites often set "no-store", "no-cache", "must-revalidate", etc., but get -upset at the traffic a spider can generate if it respects those +upset at the traffic a spider can generate if it actually respects those directives. This allows to selectively ignore Cache-Control directives that are known to be unimportant for the sites being crawled. diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 65b652953..0010b2a8f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -26,7 +26,7 @@ class HttpCompressionMiddleware(object): def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', - b",".join(ACCEPTED_ENCODINGS)) + b", ".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index dbc605a4c..7ab5b6e62 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -84,6 +84,6 @@ class RetryMiddleware(object): return retryreq else: stats.inc_value('retry/max_reached') - logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", + logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 4061d1ea3..72775df5c 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,14 +2,15 @@ import logging import inspect from scrapy.spiders import Spider -from scrapy.utils.misc import arg_to_iter +from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.misc import arg_to_iter logger = logging.getLogger(__name__) def iterate_spider_output(result): - return arg_to_iter(result) + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/tests/spiders.py b/tests/spiders.py index 39c8da0b6..3b1ee94b8 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,14 +1,18 @@ """ Some spiders used for testing and benchmarking """ +import asyncio import time from urllib.parse import urlencode +from twisted.internet import defer + from scrapy.http import Request from scrapy.item import Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): @@ -83,6 +87,36 @@ class SimpleSpider(MetaSpider): self.logger.info("Got response %d" % response.status) +class AsyncDefSpider(SimpleSpider): + + name = 'asyncdef' + + async def parse(self, response): + await defer.succeed(42) + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioSpider(SimpleSpider): + + name = 'asyncdef_asyncio' + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d" % status) + + +class AsyncDefAsyncioReturnSpider(SimpleSpider): + + name = 'asyncdef_asyncio_return' + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d" % status) + return [{'id': 1}, {'id': 2}] + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9896058dc..fca314be7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -4,16 +4,19 @@ from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse +from pytest import mark from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import signals from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, - SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback) + SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback, + AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider) class CrawlTestCase(TestCase): @@ -312,6 +315,38 @@ with multiples lines self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + @defer.inlineCallbacks + def test_async_def_parse(self): + self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + with LogCapture() as log: + yield self.runner.join() + self.assertIn("Got response 200", str(log)) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse(self): + runner = CrawlerRunner({"ASYNCIO_REACTOR": True}) + runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + with LogCapture() as log: + yield runner.join() + self.assertIn("Got response 200", str(log)) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse_list(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + self.assertIn({'id': 1}, items) + self.assertIn({'id': 2}, items) + @defer.inlineCallbacks def test_dns_server_ip_address(self): crawler = self.runner.create_crawler(SingleRequestSpider) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c6a823b53..64488841a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -48,7 +48,7 @@ class HttpCompressionTest(TestCase): } response = Response('http://scrapytest.org/', body=body, headers=headers) - response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip,deflate'}) + response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response def test_process_request(self): @@ -56,7 +56,7 @@ class HttpCompressionTest(TestCase): assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) self.assertEqual(request.headers.get('Accept-Encoding'), - b','.join(ACCEPTED_ENCODINGS)) + b', '.join(ACCEPTED_ENCODINGS)) def test_process_response_gzip(self): response = self._getresponse('gzip')