RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Crawl rule: remove deprecated code 

Remove the compatibility layer that handles 'process_request'
methods that do not receive a 'response' parameter
This commit is contained in:
Eugenio Lacuesta 2020-09-29 23:37:28 -03:00
parent 5a38639359
commit 894b509d7a
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 30 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
scrapy/spiders/crawl.py
View File

@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst
"""
import copy
import warnings
from typing import Sequence
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils.python import get_func_args
from scrapy.utils.spider import iterate_spider_output
@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor()
class Rule:
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
process_links=None, process_request=None, errback=None):
def __init__(
self,
link_extractor=None,
callback=None,
cb_kwargs=None,
follow=None,
process_links=None,
process_request=None,
errback=None,
):
self.link_extractor = link_extractor or _default_link_extractor
self.callback = callback
self.errback = errback
self.cb_kwargs = cb_kwargs or {}
self.process_links = process_links or _identity
self.process_request = process_request or _identity_process_request
self.process_request_argcount = None
self.follow = follow if follow is not None else not callback
def _compile(self, spider):
@ -53,22 +57,6 @@ class Rule:
self.errback = _get_method(self.errback, spider)
self.process_links = _get_method(self.process_links, spider)
self.process_request = _get_method(self.process_request, spider)
self.process_request_argcount = len(get_func_args(self.process_request))
if self.process_request_argcount == 1:
warnings.warn(
"Rule.process_request should accept two arguments "
"(request, response), accepting only one is deprecated",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
def _process_request(self, request, response):
"""
Wrapper around the request processing function to maintain backward
compatibility with functions that do not take a Response object
"""
args = [request] if self.process_request_argcount == 1 else [request, response]
return self.process_request(*args)
class CrawlSpider(Spider):
@ -111,7 +99,7 @@ class CrawlSpider(Spider):
for link in rule.process_links(links):
seen.add(link)
request = self._build_request(rule_index, link)
yield rule._process_request(request, response)
yield rule.process_request(request, response)
def _callback(self, response):
rule = self._rules[response.meta['rule']]

45
tests/test_spider.py
View File

@ -1,8 +1,8 @@
import gzip
import inspect
from unittest import mock
import warnings
from io import BytesIO
from unittest import mock
from testfixtures import LogCapture
from twisted.trial import unittest
@ -20,7 +20,6 @@ from scrapy.spiders import (
XMLFeedSpider,
)
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.test import get_crawler
@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest):
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
def process_request_change_domain(request):
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace('.org', '.com'))
class _CrawlSpider(self.spider_class):
@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest):
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
with warnings.catch_warnings(record=True) as cw:
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.com/somepage/item/12.html',
'http://example.com/about.html',
'http://example.com/nofollow.html'])
self.assertEqual(len(cw), 1)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.com/somepage/item/12.html',
'http://example.com/about.html',
'http://example.com/nofollow.html'])
def test_process_request_with_response(self):
@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest):
Rule(LinkExtractor(), process_request='process_request_upper'),
)
def process_request_upper(self, request):
def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())
with warnings.catch_warnings(record=True) as cw:
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
self.assertEqual(len(cw), 1)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
'http://EXAMPLE.ORG/ABOUT.HTML',
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
def test_process_request_instance_method_with_response(self):