RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Drop python 3.5 support from this PR.

This commit is contained in:
Andrey Rakhmatullin 2020-08-26 19:23:15 +05:00
parent 2f5cabc134
commit c0b2488213
19 changed files with 110 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
conftest.py
View File

@ -1,4 +1,3 @@
import sys
from pathlib import Path
import pytest
@ -15,15 +14,8 @@ collect_ignore = [
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
if sys.version_info < (3, 6):
# not importable on 3.5
collect_ignore.append("scrapy/utils/asyncgen.py")
collect_ignore.append("scrapy/utils/py36.py")
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':

4
docs/topics/coroutines.rst
View File

@ -137,9 +137,7 @@ A better behavior, which should treat requests from
requests, is enabled when :meth:`~scrapy.spiders.Spider.start_requests` is an
async function (declared using ``async def``). It doesn't need to contain
``await`` for this to work, so if you want the new queue behavior, you can just
change ``def`` to ``async def``. Note though, that using ``yield`` statements
inside an async function makes it an async generator which are only supported
since Python 3.6.
change ``def`` to ``async def``.
Usage
=====

13
docs/topics/spider-middleware.rst
View File

@ -159,13 +159,12 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
It receives an iterable (in the ``start_requests`` parameter) and must
return another iterable of :class:`~scrapy.http.Request` objects.
When using Python 3.6+ ``start_requests`` may be an
:term:`python:asynchronous iterable`. To support such spiders, this
middleware method should be an :term:`python:asynchronous generator`,
which should support both synchronous and asynchronous
``start_requests``. :func:`scrapy.utils.asyncgen.as_async_generator`
can be used to convert both kinds of ``start_requests`` into an
asynchronous iterable.
``start_requests`` may be an :term:`python:asynchronous iterable`. To
support such spiders, this middleware method should be an
:term:`python:asynchronous generator`, which should support both
synchronous and asynchronous ``start_requests``.
:func:`scrapy.utils.asyncgen.as_async_generator` can be used to convert
both kinds of ``start_requests`` into an asynchronous iterable.
.. note:: When implementing this method in your spider middleware, you
should always return an iterable (that follows the input one) and

7
scrapy/core/engine.py
View File

@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
For more information see docs/topics/architecture.rst
"""
import inspect
import logging
from time import time
@ -14,7 +15,7 @@ from scrapy import signals
from scrapy.core.scraper import Scraper
from scrapy.exceptions import DontCloseSpider
from scrapy.http import Response, Request
from scrapy.utils.defer import deferred_from_coro, _isasyncgen
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.misc import load_object
from scrapy.utils.reactor import CallLaterOnce
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
@ -27,7 +28,7 @@ class Slot:
def __init__(self, start_requests, close_if_idle, nextcall, scheduler, new_queue_behavior=False):
self.closing = False
self.inprogress = set() # requests in progress
if _isasyncgen(start_requests):
if inspect.isasyncgen(start_requests):
self.start_requests = start_requests
else:
self.start_requests = iter(start_requests)
@ -126,7 +127,7 @@ class ExecutionEngine:
@defer.inlineCallbacks
def _schedule_next_req(self, spider, slot):
try:
if _isasyncgen(slot.start_requests):
if inspect.isasyncgen(slot.start_requests):
request = yield deferred_from_coro(slot.start_requests.__anext__())
else:
request = next(slot.start_requests)

5
scrapy/crawler.py
View File

@ -98,8 +98,7 @@ class Crawler:
raise
def call_start_requests(self):
if hasattr(inspect, 'isasyncgenfunction') and inspect.isasyncgenfunction(self.spider.start_requests):
# requires Python 3.6+
if inspect.isasyncgenfunction(self.spider.start_requests):
return self.spider.start_requests().__aiter__()
elif inspect.iscoroutinefunction(self.spider.start_requests):
return deferred_from_coro(self.spider.start_requests())
@ -108,7 +107,7 @@ class Crawler:
@staticmethod
def is_start_requests_async(start_requests_function):
if hasattr(inspect, 'isasyncgenfunction') and inspect.isasyncgenfunction(start_requests_function):
if inspect.isasyncgenfunction(start_requests_function):
return True
if inspect.iscoroutinefunction(start_requests_function):
return True

3
scrapy/utils/asyncgen.py
View File

@ -1,6 +1,3 @@
"""
Helpers using Python 3.6+ async generator syntax (ignore SyntaxError on import).
"""
import collections

7
scrapy/utils/defer.py
View File

@ -173,10 +173,3 @@ def maybeDeferred_coro(f, *args, **kw):
return defer.fail(result)
else:
return defer.succeed(result)
def _isasyncgen(o):
""" Returns inspect.isasyncgen() result if it's available (requires Python 3.6),
otherwise returns False.
"""
return hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(o)

9
scrapy/utils/spider.py
View File

@ -2,19 +2,16 @@ import inspect
import logging
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro, _isasyncgen
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.misc import arg_to_iter
try:
from scrapy.utils.asyncgen import collect_asyncgen
except SyntaxError:
collect_asyncgen = None
logger = logging.getLogger(__name__)
def iterate_spider_output(result):
if collect_asyncgen and _isasyncgen(result):
if inspect.isasyncgen(result):
d = deferred_from_coro(collect_asyncgen(result))
d.addCallback(iterate_spider_output)
return d

10
tests/middlewares.py
View File

@ -1,15 +1,15 @@
import inspect
from scrapy.http import Request
from scrapy.utils.defer import _isasyncgen
class RequestInOrderMiddleware:
def process_spider_output(self, response, result, spider):
return (self._preserve_in_order(r, spider) for r in result or ())
def process_start_requests(self, start_requests, spider):
if _isasyncgen(start_requests):
from tests.py36.middlewares import RequestInOrderMiddleware_process_start_requests
return RequestInOrderMiddleware_process_start_requests(self._preserve_in_order, start_requests, spider)
async def process_start_requests(self, start_requests, spider):
if inspect.isasyncgen(start_requests):
return (self._preserve_in_order(r, spider) async for r in start_requests or ())
else:
return (self._preserve_in_order(r, spider) for r in start_requests or ())

63
tests/py36/_test_crawl.py
View File

@ -1,63 +0,0 @@
import asyncio
from scrapy import Request
from tests.spiders import SimpleSpider, YieldingRequestsSpider
class AsyncDefAsyncioGenSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen'
async def parse(self, response):
await asyncio.sleep(0.2)
yield {'foo': 42}
self.logger.info("Got response %d" % response.status)
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen_loop'
async def parse(self, response):
for i in range(10):
await asyncio.sleep(0.1)
yield {'foo': i}
self.logger.info("Got response %d" % response.status)
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen_complex'
initial_reqs = 4
following_reqs = 3
depth = 2
def _get_req(self, index, cb=None):
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
meta={'index': index},
dont_filter=True,
callback=cb)
def start_requests(self):
for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
async def parse(self, response):
index = response.meta['index']
yield {'index': index}
if index < 10 ** self.depth:
for new_index in range(10 * index, 10 * index + self.following_reqs):
yield self._get_req(new_index)
yield self._get_req(index, cb=self.parse2)
await asyncio.sleep(0.1)
yield {'index': index + 5}
async def parse2(self, response):
await asyncio.sleep(0.1)
yield {'index2': response.meta['index']}
class EagerAsyncGenSpider(YieldingRequestsSpider):
async def start_requests(self):
for r in super().start_requests():
yield r

19
tests/py36/_test_engine.py
View File

@ -1,19 +0,0 @@
#coding: utf-8
import asyncio
from scrapy import Request
from tests.test_engine import TestSpider
class StartRequestsAsyncGenSpider(TestSpider):
async def start_requests(self):
for url in self.start_urls:
yield Request(url, dont_filter=True)
class StartRequestsAsyncGenAsyncioSpider(TestSpider):
async def start_requests(self):
for url in self.start_urls:
yield Request(url, dont_filter=True)
await asyncio.sleep(1)

8
tests/py36/_test_spidermiddleware.py
View File

@ -1,8 +0,0 @@
# coding: utf-8
from scrapy.utils.asyncgen import as_async_generator
class ProcessStartRequestsAsyncGenMiddleware:
async def process_start_requests(self, start_requests, spider):
async for r in as_async_generator(start_requests):
yield r

3
tests/py36/_test_utils_asyncgen.py
View File

@ -1,3 +0,0 @@
async def async_gen():
for i in range(3):
yield i

3
tests/py36/middlewares.py
View File

@ -1,3 +0,0 @@
async def RequestInOrderMiddleware_process_start_requests(f, start_requests, spider):
async for r in start_requests or ():
yield f(r, spider)

53
tests/spiders.py
View File

@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
return reqs
class AsyncDefAsyncioGenSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen'
async def parse(self, response):
await asyncio.sleep(0.2)
yield {'foo': 42}
self.logger.info("Got response %d" % response.status)
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen_loop'
async def parse(self, response):
for i in range(10):
await asyncio.sleep(0.1)
yield {'foo': i}
self.logger.info("Got response %d" % response.status)
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
name = 'asyncdef_asyncio_gen_complex'
initial_reqs = 4
following_reqs = 3
depth = 2
def _get_req(self, index, cb=None):
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
meta={'index': index},
dont_filter=True,
callback=cb)
def start_requests(self):
for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
async def parse(self, response):
index = response.meta['index']
yield {'index': index}
if index < 10 ** self.depth:
for new_index in range(10 * index, 10 * index + self.following_reqs):
yield self._get_req(new_index)
yield self._get_req(index, cb=self.parse2)
await asyncio.sleep(0.1)
yield {'index': index + 5}
async def parse2(self, response):
await asyncio.sleep(0.1)
yield {'index2': response.meta['index']}
class ItemSpider(FollowAllSpider):
name = 'item'

13
tests/test_crawl.py
View File

@ -20,6 +20,9 @@ from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (
AsyncDefAsyncioGenComplexSpider,
AsyncDefAsyncioGenLoopSpider,
AsyncDefAsyncioGenSpider,
AsyncDefAsyncioReqsReturnSpider,
AsyncDefAsyncioReturnSingleElementSpider,
AsyncDefAsyncioReturnSpider,
@ -210,10 +213,13 @@ class CrawlTestCase(TestCase):
yield self._test_start_requests_eagerness(EagerAsyncDefSpider)
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@defer.inlineCallbacks
def test_start_requests_eagerness_asyncgen(self):
from tests.py36._test_crawl import EagerAsyncGenSpider
class EagerAsyncGenSpider(YieldingRequestsSpider):
async def start_requests(self):
for r in super().start_requests():
yield r
yield self._test_start_requests_eagerness(EagerAsyncGenSpider)
@defer.inlineCallbacks
@ -475,7 +481,6 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncgen_parse(self):
from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@ -492,7 +497,6 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
@ -512,7 +516,6 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver)

19
tests/test_engine.py
View File

@ -9,7 +9,7 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
import asyncio
import os
import re
import sys
@ -123,6 +123,19 @@ class StartRequestsAsyncDefSpider(TestSpider):
return [Request(url, dont_filter=True) for url in self.start_urls]
class StartRequestsAsyncGenSpider(TestSpider):
async def start_requests(self):
for url in self.start_urls:
yield Request(url, dont_filter=True)
class StartRequestsAsyncGenAsyncioSpider(TestSpider):
async def start_requests(self):
for url in self.start_urls:
yield Request(url, dont_filter=True)
await asyncio.sleep(1)
def start_test_site(debug=False):
root_dir = os.path.join(tests_datadir, "test_site")
r = static.File(root_dir)
@ -272,19 +285,15 @@ class EngineTest(unittest.TestCase):
yield self.run.run()
self._assert_visited_urls()
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@defer.inlineCallbacks
def test_crawler_startrequests_asyncgen(self):
from tests.py36._test_engine import StartRequestsAsyncGenSpider
self.run = CrawlerRun(StartRequestsAsyncGenSpider)
yield self.run.run()
self._assert_visited_urls()
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@mark.only_asyncio()
@defer.inlineCallbacks
def test_crawler_startrequests_asyncgen_asyncio(self):
from tests.py36._test_engine import StartRequestsAsyncGenAsyncioSpider
self.run = CrawlerRun(StartRequestsAsyncGenAsyncioSpider)
yield self.run.run()
self._assert_visited_urls()

21
tests/test_spidermiddleware.py
View File

@ -1,9 +1,7 @@
import collections
import inspect
import sys
from unittest import mock
from pytest import mark
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
@ -11,10 +9,11 @@ from twisted.python.failure import Failure
from scrapy.spiders import Spider
from scrapy.http import Request, Response
from scrapy.exceptions import _InvalidOutput
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.test import get_crawler
from scrapy.core.spidermw import SpiderMiddlewareManager
from tests.test_engine import StartRequestsAsyncDefSpider
from tests.test_engine import StartRequestsAsyncDefSpider, StartRequestsAsyncGenSpider
class SpiderMiddlewareTestCase(TestCase):
@ -121,6 +120,12 @@ class ProcessStartRequestsAsyncDefMiddleware:
return start_requests
class ProcessStartRequestsAsyncGenMiddleware:
async def process_start_requests(self, start_requests, spider):
async for r in as_async_generator(start_requests):
yield r
class ProcessStartRequestsSimple(TestCase):
""" process_start_requests tests for simple start_requests"""
@ -152,7 +157,6 @@ class ProcessStartRequestsSimple(TestCase):
@defer.inlineCallbacks
def _test_asyncgen_base(self, *mw_classes):
from scrapy.utils.asyncgen import collect_asyncgen
processed_start_requests = yield self._get_processed_start_requests(*mw_classes)
self.assertTrue(inspect.isasyncgen(processed_start_requests))
start_requests_list = yield deferred_from_coro(collect_asyncgen(processed_start_requests))
@ -169,26 +173,20 @@ class ProcessStartRequestsSimple(TestCase):
""" Async def mw """
yield self._test_simple_base(ProcessStartRequestsAsyncDefMiddleware)
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@defer.inlineCallbacks
def test_asyncgen(self):
""" Asyncgen mw """
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware)
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@defer.inlineCallbacks
def test_simple_asyncgen(self):
""" Simple mw -> asyncgen mw """
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware,
ProcessStartRequestsSimpleMiddleware)
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@defer.inlineCallbacks
def test_asyncgen_simple(self):
""" Asyncgen mw -> simple mw; cannot work """
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
processed_start_requests = yield self._get_processed_start_requests(
ProcessStartRequestsSimpleMiddleware,
ProcessStartRequestsAsyncGenMiddleware)
@ -202,13 +200,11 @@ class ProcessStartRequestsAsyncDef(ProcessStartRequestsSimple):
spider_cls = StartRequestsAsyncDefSpider
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple):
""" process_start_requests tests for async generator start_requests """
def __init__(self, methodName='runTest'):
super().__init__(methodName)
from tests.py36._test_engine import StartRequestsAsyncGenSpider
self.spider_cls = StartRequestsAsyncGenSpider
@defer.inlineCallbacks
@ -227,7 +223,6 @@ class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple):
@defer.inlineCallbacks
def test_simple_asyncgen(self):
""" Simple mw -> asyncgen mw; cannot work """
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
processed_start_requests = yield self._get_processed_start_requests(
ProcessStartRequestsAsyncGenMiddleware,
ProcessStartRequestsSimpleMiddleware)

16
tests/test_utils_asyncgen.py
View File

@ -1,17 +1,18 @@
import sys
from pytest import mark
from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import deferred_f_from_coro_f
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
async def async_gen():
for i in range(3):
yield i
class AsyncGeneratorTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_as_async_generator_simple(self):
from scrapy.utils.asyncgen import as_async_generator
gen = (i for i in range(3))
results = []
async for i in as_async_generator(gen):
@ -20,7 +21,6 @@ class AsyncGeneratorTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_as_async_generator_list(self):
from scrapy.utils.asyncgen import as_async_generator
L = [i for i in range(3)]
results = []
async for i in as_async_generator(L):
@ -29,8 +29,6 @@ class AsyncGeneratorTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_as_async_generator_async(self):
from scrapy.utils.asyncgen import as_async_generator
from tests.py36._test_utils_asyncgen import async_gen
results = []
async for i in as_async_generator(async_gen()):
results.append(i)
@ -38,7 +36,5 @@ class AsyncGeneratorTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_collect_asyncgen(self):
from scrapy.utils.asyncgen import collect_asyncgen
from tests.py36._test_utils_asyncgen import async_gen
results = await collect_asyncgen(async_gen())
self.assertEqual(results, [0, 1, 2])