mirror of https://github.com/scrapy/scrapy.git
commit
2944894c4d
|
|
@ -4700,7 +4700,7 @@ Scrapy 0.22.1 (released 2014-02-08)
|
|||
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
|
||||
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
|
||||
- Fix tests for Travis-CI build (:commit:`76c7e20`)
|
||||
- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
|
||||
- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
|
||||
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
|
||||
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
|
||||
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
|
||||
|
|
@ -4725,7 +4725,7 @@ Enhancements
|
|||
- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
|
||||
To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
|
||||
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
|
||||
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
|
||||
- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`)
|
||||
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
|
||||
- Selectors register EXSLT namespaces by default (:issue:`472`)
|
||||
- Unify item loaders similar to selectors renaming (:issue:`461`)
|
||||
|
|
@ -4905,7 +4905,7 @@ Scrapy 0.18.0 (released 2013-08-09)
|
|||
-----------------------------------
|
||||
|
||||
- Lot of improvements to testsuite run using Tox, including a way to test on pypi
|
||||
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
|
||||
- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`)
|
||||
- Use lxml recover option to parse sitemaps (:issue:`347`)
|
||||
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
|
||||
- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
|
||||
|
|
@ -4939,8 +4939,8 @@ Scrapy 0.18.0 (released 2013-08-09)
|
|||
- Added ``--pdb`` option to ``scrapy`` command line tool
|
||||
- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
|
||||
- Several improvements to spider contracts
|
||||
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
|
||||
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
|
||||
- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections,
|
||||
- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62
|
||||
- added from_crawler method to spiders
|
||||
- added system tests with mock server
|
||||
- more improvements to macOS compatibility (thanks Alex Cepoi)
|
||||
|
|
@ -5082,7 +5082,7 @@ Scrapy changes:
|
|||
- promoted :ref:`topics-djangoitem` to main contrib
|
||||
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
|
||||
- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
|
||||
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
|
||||
- removed signal: ``scrapy.mail.mail_sent``
|
||||
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
|
||||
- DBM is now the default storage backend for HTTP cache middleware
|
||||
|
|
@ -5148,7 +5148,7 @@ Scrapy 0.14
|
|||
New features and settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Support for `AJAX crawleable urls`_
|
||||
- Support for `AJAX crawlable urls`_
|
||||
- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
|
||||
- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
|
||||
- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
|
||||
|
|
@ -5408,7 +5408,7 @@ Backward-incompatible changes
|
|||
- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
|
||||
- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
|
||||
- Refactored HTTP Cache middleware
|
||||
- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
|
||||
- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
|
||||
- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
|
||||
- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
|
||||
- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
|
||||
|
|
@ -5419,7 +5419,7 @@ Scrapy 0.7
|
|||
First release of Scrapy.
|
||||
|
||||
|
||||
.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
|
||||
.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ scrapy.Spider
|
|||
.. attribute:: crawler
|
||||
|
||||
This attribute is set by the :meth:`from_crawler` class method after
|
||||
initializating the class, and links to the
|
||||
initializing the class, and links to the
|
||||
:class:`~scrapy.crawler.Crawler` object to which this spider instance is
|
||||
bound.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
A spider that generate light requests to meassure QPS throughput
|
||||
A spider that generate light requests to measure QPS throughput
|
||||
|
||||
usage:
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
|
|||
|
||||
|
||||
def _iter_command_classes(module_name):
|
||||
# TODO: add `name` attribute to commands and and merge this function with
|
||||
# TODO: add `name` attribute to commands and merge this function with
|
||||
# scrapy.utils.spider.iter_spider_classes
|
||||
for module in walk_modules(module_name):
|
||||
for obj in vars(module).values():
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
# This class used to inherit from Twisted’s
|
||||
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
|
||||
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
|
||||
# non-overriden code into this class.
|
||||
# non-overridden code into this class.
|
||||
class ScrapyHTTPClientFactory(ClientFactory):
|
||||
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ class RFC2616Policy:
|
|||
if response.status in (300, 301, 308):
|
||||
return self.MAXAGE
|
||||
|
||||
# Insufficient information to compute fresshness lifetime
|
||||
# Insufficient information to compute freshness lifetime
|
||||
return 0
|
||||
|
||||
def _compute_current_age(self, response, request, now):
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ def _select_value(ele: SelectElement, n: str, v: str):
|
|||
o = ele.value_options
|
||||
return (n, o[0]) if o else (None, None)
|
||||
if v is not None and multiple:
|
||||
# This is a workround to bug in lxml fixed 2.3.1
|
||||
# This is a workaround to bug in lxml fixed 2.3.1
|
||||
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
|
||||
selected_options = ele.xpath('.//option[@selected]')
|
||||
values = [(o.get('value') or o.text or '').strip() for o in selected_options]
|
||||
|
|
|
|||
|
|
@ -147,8 +147,8 @@ class ImagesPipeline(FilesPipeline):
|
|||
if self._deprecated_convert_image is None:
|
||||
self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image)
|
||||
if self._deprecated_convert_image:
|
||||
warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, '
|
||||
'overriden method does not accept response_body argument.',
|
||||
warnings.warn(f'{self.__class__.__name__}.convert_image() method overridden in a deprecated way, '
|
||||
'overridden method does not accept response_body argument.',
|
||||
category=ScrapyDeprecationWarning)
|
||||
|
||||
if self._deprecated_convert_image:
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ def get_meta_refresh(
|
|||
response: "scrapy.http.response.text.TextResponse",
|
||||
ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'),
|
||||
) -> Union[Tuple[None, None], Tuple[float, str]]:
|
||||
"""Parse the http-equiv refrsh parameter from the given response"""
|
||||
"""Parse the http-equiv refresh parameter from the given response"""
|
||||
if response not in _metaref_cache:
|
||||
text = response.text[0:4096]
|
||||
_metaref_cache[response] = html.get_meta_refresh(
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ def parse_url(url, encoding=None):
|
|||
|
||||
def escape_ajax(url):
|
||||
"""
|
||||
Return the crawleable url according to:
|
||||
Return the crawlable url according to:
|
||||
https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
|
||||
|
||||
>>> escape_ajax("www.example.com/ajax.html#!key=value")
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ Another example could be for building URL canonicalizers:
|
|||
::
|
||||
|
||||
#!python
|
||||
class CanonializeUrl(LegSpider):
|
||||
class CanonicalizeUrl(LegSpider):
|
||||
|
||||
def process_request(self, request):
|
||||
curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules)
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ Another example could be for building URL canonicalizers:
|
|||
::
|
||||
|
||||
#!python
|
||||
class CanonializeUrl(object):
|
||||
class CanonicalizeUrl(object):
|
||||
|
||||
def process_request(self, request, response, spider):
|
||||
curl = canonicalize_url(request.url,
|
||||
|
|
@ -594,18 +594,18 @@ A middleware to Scrape data using Parsley as described in UsingParsley
|
|||
|
||||
class ParsleyExtractor(object):
|
||||
|
||||
def __init__(self, parslet_json_code):
|
||||
parslet = json.loads(parselet_json_code)
|
||||
def __init__(self, parsley_json_code):
|
||||
parsley = json.loads(parselet_json_code)
|
||||
class ParsleyItem(Item):
|
||||
def __init__(self, *a, **kw):
|
||||
for name in parslet.keys():
|
||||
for name in parsley.keys():
|
||||
self.fields[name] = Field()
|
||||
super(ParsleyItem, self).__init__(*a, **kw)
|
||||
self.item_class = ParsleyItem
|
||||
self.parsley = PyParsley(parslet, output='python')
|
||||
self.parsley = PyParsley(parsley, output='python')
|
||||
|
||||
def process_response(self, response, request, spider):
|
||||
return self.item_class(self.parsly.parse(string=response.body))
|
||||
return self.item_class(self.parsley.parse(string=response.body))
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ If it raises an exception, Scrapy will print it and exit.
|
|||
Examples::
|
||||
|
||||
def addon_configure(settings):
|
||||
settings.overrides['DOWNLADER_MIDDLEWARES'].update({
|
||||
settings.overrides['DOWNLOADER_MIDDLEWARES'].update({
|
||||
'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline:
|
|||
return item
|
||||
|
||||
|
||||
class ProcessWithZeroDivisionErrorPipiline:
|
||||
class ProcessWithZeroDivisionErrorPipeline:
|
||||
|
||||
def process_item(self, item, spider):
|
||||
1 / 0
|
||||
|
|
|
|||
|
|
@ -316,7 +316,7 @@ class StartprojectTemplatesTest(ProjectTest):
|
|||
self.assertEqual(actual_permissions, expected_permissions)
|
||||
|
||||
def test_startproject_permissions_unchanged_in_destination(self):
|
||||
"""Check that pre-existing folders and files in the destination folder
|
||||
"""Check that preexisting folders and files in the destination folder
|
||||
do not see their permissions modified."""
|
||||
scrapy_path = scrapy.__path__[0]
|
||||
project_template = Path(scrapy_path, 'templates', 'project')
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):
|
|||
raise unittest.SkipTest("Non-existing hosts are resolvable")
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as log:
|
||||
# try to fetch the homepage of a non-existent domain
|
||||
# try to fetch the homepage of a nonexistent domain
|
||||
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
|
||||
self._assert_retried(log)
|
||||
|
||||
|
|
@ -177,7 +177,7 @@ class CrawlTestCase(TestCase):
|
|||
self.assertIs(record.exc_info[0], ZeroDivisionError)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_start_requests_lazyness(self):
|
||||
def test_start_requests_laziness(self):
|
||||
settings = {"CONCURRENT_REQUESTS": 1}
|
||||
crawler = get_crawler(BrokenStartRequestsSpider, settings)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
|
|
|
|||
|
|
@ -1067,8 +1067,8 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_ftp_download_notexist(self):
|
||||
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt",
|
||||
def test_ftp_download_nonexistent(self):
|
||||
request = Request(url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt",
|
||||
meta=self.req_meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class UserAgentMiddlewareTest(TestCase):
|
|||
self.assertEqual(req.headers['User-Agent'], b'default_useragent')
|
||||
|
||||
def test_remove_agent(self):
|
||||
# settings UESR_AGENT to None should remove the user agent
|
||||
# settings USER_AGENT to None should remove the user agent
|
||||
spider, mw = self.get_spider_and_mw('default_useragent')
|
||||
spider.user_agent = None
|
||||
mw.spider_opened(spider)
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ class DataClassItemsSpider(TestSpider):
|
|||
class ItemZeroDivisionErrorSpider(TestSpider):
|
||||
custom_settings = {
|
||||
"ITEM_PIPELINES": {
|
||||
"tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300,
|
||||
"tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -355,7 +355,7 @@ class RequestTest(unittest.TestCase):
|
|||
)
|
||||
self.assertEqual(r.method, "DELETE")
|
||||
|
||||
# If `ignore_unknon_options` is set to `False` it raises an error with
|
||||
# If `ignore_unknown_options` is set to `False` it raises an error with
|
||||
# the unknown options: --foo and -z
|
||||
self.assertRaises(
|
||||
ValueError,
|
||||
|
|
@ -873,7 +873,7 @@ class FormRequestTest(RequestTest):
|
|||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
|
||||
|
||||
def test_from_response_formname_notexist(self):
|
||||
def test_from_response_formname_nonexistent(self):
|
||||
response = _buildresponse(
|
||||
"""<form name="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
|
|
@ -912,7 +912,7 @@ class FormRequestTest(RequestTest):
|
|||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
|
||||
|
||||
def test_from_response_formname_notexists_fallback_formid(self):
|
||||
def test_from_response_formname_nonexistent_fallback_formid(self):
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
|
|
@ -927,7 +927,7 @@ class FormRequestTest(RequestTest):
|
|||
fs = _qs(r1)
|
||||
self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']})
|
||||
|
||||
def test_from_response_formid_notexist(self):
|
||||
def test_from_response_formid_nonexistent(self):
|
||||
response = _buildresponse(
|
||||
"""<form id="form1" action="post.php" method="POST">
|
||||
<input type="hidden" name="one" value="1">
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
def test_bom_is_removed_from_body(self):
|
||||
# Inferring encoding from body also cache decoded body as sideeffect,
|
||||
# this test tries to ensure that calling response.encoding and
|
||||
# response.text in indistint order doesn't affect final
|
||||
# response.text in indistinct order doesn't affect final
|
||||
# values for encoding and decoded body.
|
||||
url = 'http://example.com'
|
||||
body = b"\xef\xbb\xbfWORD"
|
||||
|
|
|
|||
|
|
@ -173,8 +173,8 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
|
||||
self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
|
||||
|
||||
expected_warning_msg = ('.convert_image() method overriden in a deprecated way, '
|
||||
'overriden method does not accept response_body argument.')
|
||||
expected_warning_msg = ('.convert_image() method overridden in a deprecated way, '
|
||||
'overridden method does not accept response_body argument.')
|
||||
self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1)
|
||||
|
||||
def test_convert_image_old(self):
|
||||
|
|
@ -182,7 +182,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter('always')
|
||||
SIZE = (100, 100)
|
||||
# straigh forward case: RGB and JPEG
|
||||
# straight forward case: RGB and JPEG
|
||||
COLOUR = (0, 127, 255)
|
||||
im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR)
|
||||
converted, _ = self.pipeline.convert_image(im)
|
||||
|
|
@ -209,14 +209,14 @@ class ImagesPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(converted.mode, 'RGB')
|
||||
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
|
||||
|
||||
# ensure that we recieved deprecation warnings
|
||||
# ensure that we received deprecation warnings
|
||||
expected_warning_msg = '.convert_image() method called in a deprecated way'
|
||||
self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4)
|
||||
|
||||
def test_convert_image_new(self):
|
||||
# tests for new API
|
||||
SIZE = (100, 100)
|
||||
# straigh forward case: RGB and JPEG
|
||||
# straight forward case: RGB and JPEG
|
||||
COLOUR = (0, 127, 255)
|
||||
im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR)
|
||||
converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
|
||||
|
|
|
|||
|
|
@ -11,12 +11,12 @@ from tests.mockserver import MockServer
|
|||
from tests.spiders import SingleRequestSpider
|
||||
|
||||
|
||||
OVERRIDEN_URL = "https://example.org"
|
||||
OVERRIDDEN_URL = "https://example.org"
|
||||
|
||||
|
||||
class ProcessResponseMiddleware:
|
||||
def process_response(self, request, response, spider):
|
||||
return response.replace(request=Request(OVERRIDEN_URL))
|
||||
return response.replace(request=Request(OVERRIDDEN_URL))
|
||||
|
||||
|
||||
class RaiseExceptionRequestMiddleware:
|
||||
|
|
@ -30,7 +30,7 @@ class CatchExceptionOverrideRequestMiddleware:
|
|||
return Response(
|
||||
url="http://localhost/",
|
||||
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
|
||||
request=Request(OVERRIDEN_URL),
|
||||
request=Request(OVERRIDDEN_URL),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
|
|||
class AlternativeCallbacksMiddleware:
|
||||
def process_response(self, request, response, spider):
|
||||
new_request = request.replace(
|
||||
url=OVERRIDEN_URL,
|
||||
url=OVERRIDDEN_URL,
|
||||
callback=spider.alt_callback,
|
||||
cb_kwargs={"foo": "bar"},
|
||||
)
|
||||
|
|
@ -127,13 +127,13 @@ class CrawlTestCase(TestCase):
|
|||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.request.url, OVERRIDEN_URL)
|
||||
self.assertEqual(response.request.url, OVERRIDDEN_URL)
|
||||
|
||||
self.assertEqual(signal_params["response"].url, url)
|
||||
self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
|
||||
self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL)
|
||||
|
||||
log.check_present(
|
||||
("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)"),
|
||||
("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)"),
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -154,7 +154,7 @@ class CrawlTestCase(TestCase):
|
|||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.body, b"Caught ZeroDivisionError")
|
||||
self.assertEqual(response.request.url, OVERRIDEN_URL)
|
||||
self.assertEqual(response.request.url, OVERRIDDEN_URL)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_do_not_override_in_process_exception(self):
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ class MixinSameOrigin:
|
|||
('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
|
||||
('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
|
||||
|
||||
# Different protocols: do NOT send refferer
|
||||
# Different protocols: do NOT send referrer
|
||||
('https://example.com/page.html', 'http://example.com/not-page.html', None),
|
||||
('https://example.com/page.html', 'http://not.example.com/', None),
|
||||
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
|
||||
|
|
@ -434,17 +434,17 @@ class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
|
|||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_NO_REFERRER}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
|
@ -561,22 +561,22 @@ class TestSettingsPolicyByName(TestCase):
|
|||
RefererMiddleware(settings)
|
||||
|
||||
|
||||
class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
|
||||
|
||||
|
||||
class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
|
||||
|
||||
|
||||
class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
class TestPolicyHeaderPrecedence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
|
||||
|
||||
|
||||
class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
class TestPolicyHeaderPrecedence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
"""
|
||||
The empty string means "no-referrer-when-downgrade"
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
|
|||
self.assertIn('foo.NewClass', str(w[1].message))
|
||||
self.assertIn('bar.OldClass', str(w[1].message))
|
||||
|
||||
def test_subclassing_warns_only_on_direct_childs(self):
|
||||
def test_subclassing_warns_only_on_direct_children(self):
|
||||
Deprecated = create_deprecated_class('Deprecated', NewName,
|
||||
warn_once=False,
|
||||
warn_category=MyWarning)
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
b = Obj()
|
||||
# no attributes given return False
|
||||
self.assertFalse(equal_attributes(a, b, []))
|
||||
# not existent attributes
|
||||
# nonexistent attributes
|
||||
self.assertFalse(equal_attributes(a, b, ['x', 'y']))
|
||||
|
||||
a.x = 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue