From f3c6bfdebe2f8ee3cfaec172ab8607f75829fd43 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:10 -0500 Subject: [PATCH 01/27] spelling: accounting Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index c97de0ed8..dd51b11da 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5082,7 +5082,7 @@ Scrapy changes: - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method -- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module +- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled - DBM is now the default storage backend for HTTP cache middleware From 334f844e58edfbd791fee6697e3259dbb5a741a5 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:09:18 -0500 Subject: [PATCH 02/27] spelling: and Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scrapy/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 8218a51c8..88936c276 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -24,7 +24,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _iter_command_classes(module_name): - # TODO: add `name` attribute to commands and and merge this function with + # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): for obj in vars(module).values(): From 226c42ad1423d92ae9b6aa6e05f4e5d5cdd9f3f8 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:11 -0500 Subject: [PATCH 03/27] spelling: canonicalize Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sep/sep-016.rst | 2 +- sep/sep-018.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-016.rst b/sep/sep-016.rst index 335f09f45..a60ab30dd 100644 --- a/sep/sep-016.rst +++ b/sep/sep-016.rst @@ -148,7 +148,7 @@ Another example could be for building URL canonicalizers: :: #!python - class CanonializeUrl(LegSpider): + class CanonicalizeUrl(LegSpider): def process_request(self, request): curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fe707923a..fa4f7da82 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -321,7 +321,7 @@ Another example could be for building URL canonicalizers: :: #!python - class CanonializeUrl(object): + class CanonicalizeUrl(object): def process_request(self, request, response, spider): curl = canonicalize_url(request.url, From 1300c1c8816c86e8ce9cff114dcefdf785138018 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 21:33:14 -0500 Subject: [PATCH 04/27] spelling: children Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_utils_deprecate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 50c63dfab..9fbb07472 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -69,7 +69,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn('foo.NewClass', str(w[1].message)) self.assertIn('bar.OldClass', str(w[1].message)) - def test_subclassing_warns_only_on_direct_childs(self): + def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class('Deprecated', NewName, warn_once=False, warn_category=MyWarning) From 87fc92441f39df4eb1a43482f1475e2ba5415130 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:11 -0500 Subject: [PATCH 05/27] spelling: crawlable Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/news.rst | 8 ++++---- scrapy/utils/url.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index dd51b11da..4609c5158 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4725,7 +4725,7 @@ Enhancements - [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) -- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) +- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) @@ -4905,7 +4905,7 @@ Scrapy 0.18.0 (released 2013-08-09) ----------------------------------- - Lot of improvements to testsuite run using Tox, including a way to test on pypi -- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) +- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) - Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) @@ -5148,7 +5148,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawleable urls`_ +- Support for `AJAX crawlable urls`_ - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -5419,7 +5419,7 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 21201ace5..6c59af6db 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -46,7 +46,7 @@ def parse_url(url, encoding=None): def escape_ajax(url): """ - Return the crawleable url according to: + Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started >>> escape_ajax("www.example.com/ajax.html#!key=value") From 581eb2d1b4fe89953115859cc6cff1cbc30bc700 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:12 -0500 Subject: [PATCH 06/27] spelling: downloader Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sep/sep-021.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sep/sep-021.rst b/sep/sep-021.rst index c1ec16f7f..d56bc26af 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -79,7 +79,7 @@ If it raises an exception, Scrapy will print it and exit. Examples:: def addon_configure(settings): - settings.overrides['DOWNLADER_MIDDLEWARES'].update({ + settings.overrides['DOWNLOADER_MIDDLEWARES'].update({ 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, }) From e6ebadcd54e018dc1f90505bcdde024dba813a80 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:12 -0500 Subject: [PATCH 07/27] spelling: freshness Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scrapy/extensions/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 4e76fe5e5..b8d8e94dc 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -190,7 +190,7 @@ class RFC2616Policy: if response.status in (300, 301, 308): return self.MAXAGE - # Insufficient information to compute fresshness lifetime + # Insufficient information to compute freshness lifetime return 0 def _compute_current_age(self, response, request, now): From b6426b8e03759f3364da88f52406aa8cf8dbf92c Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:13 -0500 Subject: [PATCH 08/27] spelling: heavily Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 4609c5158..ee7462d5c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5408,7 +5408,7 @@ Backward-incompatible changes - Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`) - Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`) - Refactored HTTP Cache middleware -- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) +- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) - Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120) - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121) - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`) From a26b6b0607026e4ea266ed3e30d2643d5990bf8a Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:13 -0500 Subject: [PATCH 09/27] spelling: indistinct Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_http_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 74e170ec0..bb16b8904 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -425,7 +425,7 @@ class TextResponseTest(BaseResponseTest): def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.text in indistint order doesn't affect final + # response.text in indistinct order doesn't affect final # values for encoding and decoded body. url = 'http://example.com' body = b"\xef\xbb\xbfWORD" From b62aacfee36397e8173cf3370a2088f3bb53943f Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:14 -0500 Subject: [PATCH 10/27] spelling: initializing Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index ffe41cf3e..7aa8555d5 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -99,7 +99,7 @@ scrapy.Spider .. attribute:: crawler This attribute is set by the :meth:`from_crawler` class method after - initializating the class, and links to the + initializing the class, and links to the :class:`~scrapy.crawler.Crawler` object to which this spider instance is bound. From e894db2f3f7d0873c38542744fde946a6db0d659 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:14 -0500 Subject: [PATCH 11/27] spelling: laziness Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5ec96e4a7..4f7c24e27 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -177,7 +177,7 @@ class CrawlTestCase(TestCase): self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks - def test_start_requests_lazyness(self): + def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) From fff2f2db20c7a147fb2631bd6deca03a74e7e208 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:15 -0500 Subject: [PATCH 12/27] spelling: measure Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- extras/qpsclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 28703650d..2b501d11d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,5 +1,5 @@ """ -A spider that generate light requests to meassure QPS throughput +A spider that generate light requests to measure QPS throughput usage: From 5f33a64a02843ca595e477756cdf970e2f944278 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:15 -0500 Subject: [PATCH 13/27] spelling: middleware Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index ee7462d5c..a1e3e25e0 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4939,8 +4939,8 @@ Scrapy 0.18.0 (released 2013-08-09) - Added ``--pdb`` option to ``scrapy`` command line tool - Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts -- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, -- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, +- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 - added from_crawler method to spiders - added system tests with mock server - more improvements to macOS compatibility (thanks Alex Cepoi) From a839b61147da92ad7b175919fb31fae8293cec97 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:09:28 -0500 Subject: [PATCH 14/27] spelling: nonexistent Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_crawl.py | 2 +- tests/test_downloader_handlers.py | 4 ++-- tests/test_http_request.py | 6 +++--- tests/test_utils_python.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4f7c24e27..44ced42f5 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -150,7 +150,7 @@ class CrawlTestCase(TestCase): raise unittest.SkipTest("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) with LogCapture() as log: - # try to fetch the homepage of a non-existent domain + # try to fetch the homepage of a nonexistent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3dc2745a0..433b6d0c8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1050,8 +1050,8 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) - def test_ftp_download_notexist(self): - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", + def test_ftp_download_nonexistent(self): + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9f7f1854f..c3b729b76 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -873,7 +873,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) - def test_from_response_formname_notexist(self): + def test_from_response_formname_nonexistent(self): response = _buildresponse( """
@@ -912,7 +912,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) - def test_from_response_formname_notexists_fallback_formid(self): + def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( """ @@ -927,7 +927,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) - def test_from_response_formid_notexist(self): + def test_from_response_formid_nonexistent(self): response = _buildresponse( """ diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 403e4f8fe..8cca17aa5 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -160,7 +160,7 @@ class UtilsPythonTestCase(unittest.TestCase): b = Obj() # no attributes given return False self.assertFalse(equal_attributes(a, b, [])) - # not existent attributes + # nonexistent attributes self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.x = 1 From 1a5cf00db78dc5173e75b85951f67c323b81ad96 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:17 -0500 Subject: [PATCH 15/27] spelling: overridden Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scrapy/core/downloader/webclient.py | 2 +- scrapy/pipelines/images.py | 4 ++-- tests/test_pipeline_images.py | 4 ++-- tests/test_request_attribute_binding.py | 16 ++++++++-------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 255ca62e6..a261bf0ad 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -98,7 +98,7 @@ class ScrapyHTTPPageGetter(HTTPClient): # This class used to inherit from Twisted’s # twisted.web.client.HTTPClientFactory. When that class was deprecated in # Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overriden code into this class. +# non-overridden code into this class. class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6a28a3b87..05b8693ad 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -147,8 +147,8 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.', + warnings.warn(f'{self.__class__.__name__}.convert_image() method overridden in a deprecated way, ' + 'overridden method does not accept response_body argument.', category=ScrapyDeprecationWarning) if self._deprecated_convert_image: diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f98d40fda..30166a502 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -173,8 +173,8 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - expected_warning_msg = ('.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.') + expected_warning_msg = ('.convert_image() method overridden in a deprecated way, ' + 'overridden method does not accept response_body argument.') self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) def test_convert_image_old(self): diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 15e400327..c1ee4baee 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -11,12 +11,12 @@ from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider -OVERRIDEN_URL = "https://example.org" +OVERRIDDEN_URL = "https://example.org" class ProcessResponseMiddleware: def process_response(self, request, response, spider): - return response.replace(request=Request(OVERRIDEN_URL)) + return response.replace(request=Request(OVERRIDDEN_URL)) class RaiseExceptionRequestMiddleware: @@ -30,7 +30,7 @@ class CatchExceptionOverrideRequestMiddleware: return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), - request=Request(OVERRIDEN_URL), + request=Request(OVERRIDDEN_URL), ) @@ -52,7 +52,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider): class AlternativeCallbacksMiddleware: def process_response(self, request, response, spider): new_request = request.replace( - url=OVERRIDEN_URL, + url=OVERRIDDEN_URL, callback=spider.alt_callback, cb_kwargs={"foo": "bar"}, ) @@ -127,13 +127,13 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, OVERRIDEN_URL) + self.assertEqual(response.request.url, OVERRIDDEN_URL) self.assertEqual(signal_params["response"].url, url) - self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL) log.check_present( - ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), + ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), ) @defer.inlineCallbacks @@ -154,7 +154,7 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, OVERRIDEN_URL) + self.assertEqual(response.request.url, OVERRIDDEN_URL) @defer.inlineCallbacks def test_downloader_middleware_do_not_override_in_process_exception(self): From f5d024f16c8c3aea6a7dc4e648e907dfa3bd496f Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:17 -0500 Subject: [PATCH 16/27] spelling: parsley Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- sep/sep-018.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fa4f7da82..96df82f6b 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -594,18 +594,18 @@ A middleware to Scrape data using Parsley as described in UsingParsley class ParsleyExtractor(object): - def __init__(self, parslet_json_code): - parslet = json.loads(parselet_json_code) + def __init__(self, parsley_json_code): + parsley = json.loads(parselet_json_code) class ParsleyItem(Item): def __init__(self, *a, **kw): - for name in parslet.keys(): + for name in parsley.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.parsley = PyParsley(parsley, output='python') def process_response(self, response, request, spider): - return self.item_class(self.parsly.parse(string=response.body)) + return self.item_class(self.parsley.parse(string=response.body)) From 8ae77fdb34d862b4ad466f7f9764132502787c47 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:18 -0500 Subject: [PATCH 17/27] spelling: pipeline Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/pipelines.py | 2 +- tests/test_engine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipelines.py b/tests/pipelines.py index fed2af7d3..c130f057f 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline: return item -class ProcessWithZeroDivisionErrorPipiline: +class ProcessWithZeroDivisionErrorPipeline: def process_item(self, item, spider): 1 / 0 diff --git a/tests/test_engine.py b/tests/test_engine.py index aa3313659..870993edf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -109,7 +109,7 @@ class DataClassItemsSpider(TestSpider): class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { "ITEM_PIPELINES": { - "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300, + "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, } } From 66ab82f12631e82c2c0426d68e52ac5e59344402 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:18 -0500 Subject: [PATCH 18/27] spelling: precedence Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_spidermiddleware_referer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 63daf0b8a..6a2af0e7d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -434,17 +434,17 @@ class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} @@ -561,22 +561,22 @@ class TestSettingsPolicyByName(TestCase): RefererMiddleware(settings) -class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} -class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} -class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPrecedence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPrecedence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): """ The empty string means "no-referrer-when-downgrade" """ From 826e0ee6111739d154176bfdabb32cbf13f1dd0c Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:09:34 -0500 Subject: [PATCH 19/27] spelling: preexisting Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 91476abf8..b8b481f20 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -316,7 +316,7 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) def test_startproject_permissions_unchanged_in_destination(self): - """Check that pre-existing folders and files in the destination folder + """Check that preexisting folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] project_template = Path(scrapy_path, 'templates', 'project') From 6aa5374bd3924dc98047a704569f5e1fc87559f1 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:19 -0500 Subject: [PATCH 20/27] spelling: received Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_pipeline_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 30166a502..f4d36aae9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -209,7 +209,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - # ensure that we recieved deprecation warnings + # ensure that we received deprecation warnings expected_warning_msg = '.convert_image() method called in a deprecated way' self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4) From d27c611cc0d2fd9786752c8996960dd23fd8e2ac Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:19 -0500 Subject: [PATCH 21/27] spelling: referrer Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_spidermiddleware_referer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 6a2af0e7d..df44a1c42 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -147,7 +147,7 @@ class MixinSameOrigin: ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - # Different protocols: do NOT send refferer + # Different protocols: do NOT send referrer ('https://example.com/page.html', 'http://example.com/not-page.html', None), ('https://example.com/page.html', 'http://not.example.com/', None), ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), From 010cf9d42089f33dbc6408f034adab289becbea8 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:20 -0500 Subject: [PATCH 22/27] spelling: refresh Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scrapy/utils/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 23bd2da65..7a6f4a96f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -36,7 +36,7 @@ def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), ) -> Union[Tuple[None, None], Tuple[float, str]]: - """Parse the http-equiv refrsh parameter from the given response""" + """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh( From 1e44d4614ec7d34c4fb8f50a3429e4b1ddeb2253 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:20 -0500 Subject: [PATCH 23/27] spelling: straight Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_pipeline_images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f4d36aae9..6f5466191 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -182,7 +182,7 @@ class ImagesPipelineTestCase(unittest.TestCase): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) @@ -216,7 +216,7 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_convert_image_new(self): # tests for new API SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) From 860fbef608445230c16d0ddd4c2b676866449862 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:21 -0500 Subject: [PATCH 24/27] spelling: unknown Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index c3b729b76..81cebdc7b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -355,7 +355,7 @@ class RequestTest(unittest.TestCase): ) self.assertEqual(r.method, "DELETE") - # If `ignore_unknon_options` is set to `False` it raises an error with + # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z self.assertRaises( ValueError, From 2cb1e6668a51294df909b0ac5f7e35b7e858446e Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 20:26:21 -0500 Subject: [PATCH 25/27] spelling: workaround Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8e0a7fae2..fd61a8e4e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -191,7 +191,7 @@ def _select_value(ele: SelectElement, n: str, v: str): o = ele.value_options return (n, o[0]) if o else (None, None) if v is not None and multiple: - # This is a workround to bug in lxml fixed 2.3.1 + # This is a workaround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') values = [(o.get('value') or o.text or '').strip() for o in selected_options] From 8a526d161c0a7d4818f16f6a9e1ae563e16abab6 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 21:46:11 -0500 Subject: [PATCH 26/27] spelling: user Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- tests/test_downloadermiddleware_useragent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index a286764fd..c97e28cb8 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -20,7 +20,7 @@ class UserAgentMiddlewareTest(TestCase): self.assertEqual(req.headers['User-Agent'], b'default_useragent') def test_remove_agent(self): - # settings UESR_AGENT to None should remove the user agent + # settings USER_AGENT to None should remove the user agent spider, mw = self.get_spider_and_mw('default_useragent') spider.user_agent = None mw.spider_opened(spider) From 41734bb5c1310760db012ff705ce98910b955791 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 18 Dec 2022 21:46:49 -0500 Subject: [PATCH 27/27] spelling: unencodeable Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index a1e3e25e0..07264827b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4700,7 +4700,7 @@ Scrapy 0.22.1 (released 2014-02-08) - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`) - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`) - Fix tests for Travis-CI build (:commit:`76c7e20`) -- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) +- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`) - Updated the tutorial crawl output with latest output. (:commit:`8da65de`) - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)