From 72cf190145196c3054b611eef7a0eef30ac63c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 14:46:07 +0100 Subject: [PATCH 01/34] Add a FAQ entry about name collisions --- docs/faq.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..8de680816 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -319,7 +319,18 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. +Running ``runspider`` I get ``error: No spider found in file: `` +-------------------------------------------------------------------------- + +This may happen if your Scrapy project has a spider module with a name that +conflicts with the name of one of the `Python standard library modules`_, such +as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed. +See :issue:`2680`. + +.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python package: https://pypi.org/ + .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file From 0770961054f24d56d219a81d9e0c467de98312c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 16:05:57 +0200 Subject: [PATCH 02/34] Write a test for #4665 --- tests/test_commands.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 002237824..2e5bd6c00 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -66,9 +66,14 @@ class ProjectTest(unittest.TestCase): def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args - p = subprocess.Popen(args, cwd=self.cwd, env=self.env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **popen_kwargs) + p = subprocess.Popen( + args, + cwd=popen_kwargs.pop('cwd', self.cwd), + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **popen_kwargs, + ) def kill_proc(): p.kill() @@ -122,6 +127,32 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject')) self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def test_existing_project_dir(self): + project_dir = mkdtemp() + os.mkdir(os.path.join(project_dir, self.project_name)) + + p, out, err = self.proc('startproject', self.project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) + + assert exists(join(abspath(project_dir), 'scrapy.cfg')) + assert exists(join(abspath(project_dir), 'testproject')) + assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + + self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) + + self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) + self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) + self.assertEqual(1, self.call('startproject', 'wrong---project---name')) + self.assertEqual(1, self.call('startproject', 'sys')) + self.assertEqual(2, self.call('startproject')) + self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def get_permissions_dict(path, renamings=None, ignore=None): renamings = renamings or tuple() From 544c1f6e390c72053f768d3992ea2d0801363b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 16:30:34 +0200 Subject: [PATCH 03/34] Fix the issue --- scrapy/commands/startproject.py | 8 ++++---- tests/test_commands.py | 34 +++++++++++++++------------------ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..35b58090c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,7 +1,7 @@ import re import os import string -from importlib import import_module +from importlib.util import find_spec from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -43,10 +43,10 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): try: - import_module(module_name) - return True - except ImportError: + spec = find_spec(module_name) + except ModuleNotFoundError: return False + return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): print('Error: Project names must begin with a letter and contain' diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e5bd6c00..10a3aa16c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -92,7 +92,10 @@ class ProjectTest(unittest.TestCase): class StartprojectTest(ProjectTest): def test_startproject(self): - self.assertEqual(0, self.call('startproject', self.project_name)) + p, out, err = self.proc('startproject', self.project_name) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) assert exists(join(self.proj_path, 'scrapy.cfg')) assert exists(join(self.proj_path, 'testproject')) @@ -129,29 +132,22 @@ class StartprojectTest(ProjectTest): def test_existing_project_dir(self): project_dir = mkdtemp() - os.mkdir(os.path.join(project_dir, self.project_name)) + project_name = self.project_name + '_existing' + project_path = os.path.join(project_dir, project_name) + os.mkdir(project_path) - p, out, err = self.proc('startproject', self.project_name, cwd=project_dir) + p, out, err = self.proc('startproject', project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(abspath(project_dir), 'scrapy.cfg')) - assert exists(join(abspath(project_dir), 'testproject')) - assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) - - self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) - - self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) - self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) - self.assertEqual(1, self.call('startproject', 'wrong---project---name')) - self.assertEqual(1, self.call('startproject', 'sys')) - self.assertEqual(2, self.call('startproject')) - self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + assert exists(join(abspath(project_path), 'scrapy.cfg')) + assert exists(join(abspath(project_path), project_name)) + assert exists(join(join(abspath(project_path), project_name), '__init__.py')) + assert exists(join(join(abspath(project_path), project_name), 'items.py')) + assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_path), project_name), 'settings.py')) + assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) def get_permissions_dict(path, renamings=None, ignore=None): From 1cc8d5829fc1b1b10fd852db693ac44dc9be0ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 6 Aug 2020 13:52:47 +0200 Subject: [PATCH 04/34] Remove unneeded try-except Exceptions only happen when find_spec gets a 2nd parameter. --- scrapy/commands/startproject.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 35b58090c..82ccda35e 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -42,10 +42,7 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): - try: - spec = find_spec(module_name) - except ModuleNotFoundError: - return False + spec = find_spec(module_name) return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): From 91f81445524630843561c8a3c71b7fa0f081d783 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 00:30:18 -0300 Subject: [PATCH 05/34] Remove deprecated Spider.make_requests_from_url method --- scrapy/spiders/__init__.py | 27 ++------------------------- tests/test_spider.py | 33 --------------------------------- 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..d8248c606 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -4,14 +4,12 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ import logging -import warnings from typing import Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import method_is_overridden class Spider(object_ref): @@ -57,34 +55,13 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - cls = self.__class__ if not self.start_urls and hasattr(self, 'start_url'): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " "did you miss an 's'?)") - if method_is_overridden(cls, Spider, 'make_requests_from_url'): - warnings.warn( - "Spider.make_requests_from_url method is deprecated; it " - "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead " - f"(see {cls.__module__}.{cls.__name__}).", - ) - for url in self.start_urls: - yield self.make_requests_from_url(url) - else: - for url in self.start_urls: - yield Request(url, dont_filter=True) - - def make_requests_from_url(self, url): - """ This method is deprecated. """ - warnings.warn( - "Spider.make_requests_from_url method is deprecated: " - "it will be removed and not be called by the default " - "Spider.start_requests method in future Scrapy releases. " - "Please override Spider.start_requests method instead." - ) - return Request(url, dont_filter=True) + for url in self.start_urls: + yield Request(url, dont_filter=True) def _parse(self, response, **kwargs): return self.parse(response, **kwargs) diff --git a/tests/test_spider.py b/tests/test_spider.py index d23543f6a..a7c3ee048 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name='foo'), Spider) - def test_make_requests_from_url_deprecated(self): - class MySpider4(Spider): - name = 'spider1' - start_urls = ['http://example.com'] - - class MySpider5(Spider): - name = 'spider2' - start_urls = ['http://example.com'] - - def make_requests_from_url(self, url): - return Request(url + "/foo", dont_filter=True) - - with warnings.catch_warnings(record=True) as w: - # spider without overridden make_requests_from_url method - # doesn't issue a warning - spider1 = MySpider4() - self.assertEqual(len(list(spider1.start_requests())), 1) - self.assertEqual(len(w), 0) - - # spider without overridden make_requests_from_url method - # should issue a warning when called directly - request = spider1.make_requests_from_url("http://www.example.com") - self.assertTrue(isinstance(request, Request)) - self.assertEqual(len(w), 1) - - # spider with overridden make_requests_from_url issues a warning, - # but the method still works - spider2 = MySpider5() - requests = list(spider2.start_requests()) - self.assertEqual(len(requests), 1) - self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 2) - class NoParseMethodSpiderTest(unittest.TestCase): From 8bbaea9892003769672204a8ff4e989b5aab5ceb Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 16:57:43 +0530 Subject: [PATCH 06/34] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 80c6c2c37..222dda685 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,6 +1073,8 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` +* only works with python 3.8 and earlier + .. _rerp-parser: Robotexclusionrulesparser From 1a8b98843aee548a52faa36f5360a81a1624e208 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:00:05 +0530 Subject: [PATCH 07/34] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 222dda685..2c00ad45d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,7 +1073,7 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* only works with python 3.8 and earlier +* Only works with python 3.8 and earlier .. _rerp-parser: From cc1cb2de0c6e91393ceb6872c174aa2ac06c07ac Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:21:47 +0530 Subject: [PATCH 08/34] updated suggested changes --- docs/topics/downloader-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2c00ad45d..fa211fb75 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,12 +1068,13 @@ Native implementation, provides better speed than Protego. In order to use this parser: +.. warning:: Does not support Python 3.9+ + * Install `Reppy `_ by running ``pip install reppy`` * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* Only works with python 3.8 and earlier .. _rerp-parser: From 013ac90f6129a9e8b862d66ca2ffa8f0f1fd674e Mon Sep 17 00:00:00 2001 From: umair ansari Date: Mon, 16 Aug 2021 18:00:06 +0530 Subject: [PATCH 09/34] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..8323bc564 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` + .. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From ebddb77a331c6290e64e449ef9847e33b323a146 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 18:08:26 +0530 Subject: [PATCH 10/34] updated suggested changes after review --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..4d7c87404 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` +.. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From bcf38a67194f25db66334af18bdf49b34c6a0c39 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:48:47 +0530 Subject: [PATCH 11/34] added upstream issue for not supported python version --- docs/topics/downloader-middleware.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 4d7c87404..089d5683a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,7 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: Does not support Python 3.9+ + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From d623ed15d1a79a91b55aa7aae2d942aac94abfdf Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:51:03 +0530 Subject: [PATCH 12/34] indentation updated --- docs/topics/downloader-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 089d5683a..928a59bf1 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. +.. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From 2d2581c68f35799dc4372a257eaa8dbb5208481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Aug 2021 12:46:42 +0200 Subject: [PATCH 13/34] Move documentation about avoiding bans into a topic of its own (#4039) --- docs/index.rst | 4 + docs/topics/avoiding-bans.rst | 340 ++++++++++++++++++++++++++++++++++ docs/topics/practices.rst | 34 ---- 3 files changed, 344 insertions(+), 34 deletions(-) create mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 433798aa8..7647b3781 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,6 +155,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +180,9 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/avoiding-bans` + Avoid getting banned from websites. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst new file mode 100644 index 000000000..59f0da191 --- /dev/null +++ b/docs/topics/avoiding-bans.rst @@ -0,0 +1,340 @@ +.. _bans: + +============= +Avoiding bans +============= + +This topic covers some of the strategies that you can follow to avoid getting +different or bad responses from a website that you are crawling due to filters +such as regional filters, web browser filters, etc. + +.. _avoiding-crawls: + +Avoiding crawls +=============== + +The best way not to be banned from a website is not to send requests to it in +the first place. + +One way to avoid crawling a website is to find the desired dataset through +other means. For example, you can use Google’s `dataset search engine`_. + +If the target website is the only or best source of the desired information, +and you only need to extract the data on a monthly basis or a lower frequency, +you may be able to crawl a public snapshot of the target website instead. +`Common Crawl`_ is an open repository of web crawl data that you can access +freely. It contains monthly snapshots of a wide variety of websites and, if you +are lucky, your target website will be among them. + +.. _Common Crawl: https://commoncrawl.org/ +.. _dataset search engine: https://datasetsearch.research.google.com/ + + +.. _being-polite: + +Being polite +============ + +To avoid being banned, you should first avoid giving a website reasons to ban +you. + +.. _identifying-yourself: + +Identifying yourself +-------------------- + +If your crawling has a noticeable negative impact on a website or you crawl +content that should not be crawled, website administrators will need to do +something. + +Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and +includes contact information, so that website administrators can contact you. + + +.. _following-robotstxt: + +Following robots.txt guidelines +------------------------------- + +Some websites provide a ``robots.txt`` file at their root path (e.g. +``http://example.com/robots.txt``) that describes the guidelines that they wish +bots to follow when crawling their website. + +Before you start writing a spider for a website, read their ``robots.txt`` +file and implement your spider following its guidelines. See the `robots.txt +standard draft`_ or the `robots.txt Google specification`_ for information on +how to read ``robots.txt`` files. + +To ensure that your spider does not crawl pages restricted by ``robots.txt`` +guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the +:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` +middleware. When you do, if your spider attempts to crawl a restricted page, +this middleware aborts that request with the following message:: + + Forbidden by robots.txt + +Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and +:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or +``Request-Rate`` directives from the ``robots.txt`` guidelines. + +You may also use the :ref:`AutoThrottle extension ` on top +of that, so that when the target website experiences a high load, your spider +automatically switches to higher download delays. + +.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt +.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 + + +.. _choosing-crawl-speed: + +Finding the right guidelines on your own +---------------------------------------- + +If a website does not specify a desired download delay, or does not provide a +``robots.txt`` file, you should make an effort to find out the right values for +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that +will not have a noticeable negative impact on the target website. + +Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that +the target website receives, and choose concurrency and delay values that will +not cause a noticeable traffic increase. + +.. _SimilarWeb: https://www.similarweb.com + + +.. _filters-and-challenges: + +Bypassing filters and solving challenges +======================================== + +Some websites implement filters and challenges that aim to deny access or alter +their content based on aspects of the visitor, such as the country where they +are or the web browsing tool they use. + +.. _regional-filter: + +Bypassing regional filters +-------------------------- + +Some websites send different or bad responses based on the region or country +associated to your `IP address`_. + +To bypass these filters, get access to a `proxy server`_ that has an outgoing +IP address from a region that gets the desired responses. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _IP address: https://en.wikipedia.org/wiki/IP_address +.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server + + +.. _web-browser-filter: + +Bypassing web browser filters +----------------------------- + +Some websites send different or bad responses if they detect that your request +does not come from a web browser. + +To bypass these filters, switch your :setting:`USER_AGENT` to a value copied +from those that popular web browsers use. In some rare cases, you may need a +user agent string from a specific web browser. + +There are multiple Scrapy plugins that can rotate your requests through popular +web browser user agent strings, such as scrapy-fake-useragent_, +scrapy-random-useragent_ or Scrapy-UserAgents_. + +For advanced web browser filters, +:ref:`pre-rendering JavaScript ` or +:ref:`using a headless browser ` may be necessary. +Use these options only as a last resort, however, because they cause a higher +load per request on the target website. + +.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent +.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent +.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ + + +.. _request-delay-filter: + +Bypassing request delay filters +------------------------------- + +Some websites may ban your IP after they detect that your requests use a +constant download delay. + +To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting is enabled by default. When that is not enough, an +:ref:`IP address rotation solution ` may be much more effective. + + +.. _isp-filter: + +Bypassing internet service provider filters +------------------------------------------- + +Some websites send different or bad responses if they detect that your request +comes from an IP address that belongs to a `data center`_, as opposed to a +residential IP address from an `internet service provider`_ or a mobile IP +address from a `mobile network`_. + +To bypass these filters, get access to a proxy server that has an outgoing IP +address that is either residential or mobile. Note that you may also get +different responses depending on whether your IP address is residential or +mobile. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _data center: https://en.wikipedia.org/wiki/Data_center +.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider +.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network + + +.. _captcha: + +Solving CAPTCHA challenges +-------------------------- + +Some websites require you to solve a `CAPTCHA challenge`_ to get the desired +response. + +To bypass these filters, several options exist: + +- You could have your spider present the CAPTCHA challenge to you and wait + for you to solve it manually. + +- Some CAPTCHA challenges can be solved using an `optical character + recognition`_ (OCR) solution such as pytesseract_. + +- Paid CAPTCHA solving services exist. + +Whichever solution you choose, implement it as a :ref:`downloader middleware +` that automatically detects CAPTCHA challenges +in responses and solves them, so that your spider code only receives successful +responses. + +.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA +.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition +.. _pytesseract: https://github.com/madmaze/pytesseract + + +.. _ip-rotation: + +IP address rotation solutions +============================= + +See below some of the different solutions there are to have your requests use +different outgoing IP addresses. + +When using this approach, remember to set :setting:`COOKIES_ENABLED` to +``False`` to disable global cookie handling. This prevents websites from +identifying two requests as coming from the same user agent even if they come +from different IP addresses and have different user-agent strings. You can +still include some cookies manually in your requests. Define them through the +``Cookies`` header of your requests. See +:class:`Request.headers `. + +.. _smart-proxy: + +Smart proxies +------------- + +An increasing number of websites use solutions that apply many of the above +filters and challenges at the same time. + +There are paid proxy services, like `Zyte Smart Proxy Manager`_, that +automatically bypass website filters and challenges, so that your spider only +gets successful responses. They also allow managing sessions to simulate user +behavior. + +For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced +integration with Scrapy. For other services, use the +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware +or implement your own :ref:`downloader middleware +`. + +.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ + + +.. _rotating-proxy: + +Rotating proxies +---------------- + +Rotating proxy services like ProxyMesh_ send different requests through +different proxies. This can decrease the likelihood of being affected by some +filters or challenges. + +.. _ProxyMesh: https://proxymesh.com/ + + +.. _free-proxies: + +Free proxies +------------ + +You can easily find lists of free proxies in the internet, and you can use +a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in +your spider and have requests rotate through them automatically. + +This approach, however, has serious drawbacks: + +- Free proxies may stop working at any moment. You need to implement a way to + refresh your list of free proxies. + +- In addition to handling occasional bad responses from websites, you + need to handle all kinds of bad responses from proxies. You may even need + to inspect the response body to determine if a response comes from the + target website or from a misbehaving proxy. + +- Advanced antibot solutions may automatically detect and filter out traffic + from free proxies. + +.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies + + +.. _custom-rotating-proxy: + +Custom rotating proxy +--------------------- + +If you have spare servers, you can set them up as proxies and use scrapoxy_ to +build a custom proxy that rotates traffic through them. However, the initial +setup can be complex, and your requests will be vulnerable to +:ref:`internet service provider filtering `. + +.. _scrapoxy: https://scrapoxy.io/ + + +.. _tor: + +The Tor network +--------------- + +It is possible to send requests through the `Tor network`_. + +The initial setup to have Scrapy working with Tor is not straightforward. +Use a search engine to find up-to-date documentation specific to using +Scrapy and Tor together. + +The main drawback of using the Tor network is that traffic can be extremely +slow. + +.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) + + +.. _commercial-support: + +Seeking professional help +========================= + +Avoiding bans, filters and challenges can be difficult and tricky, and may +sometimes require special infrastructure. + +If you find yourself unable to prevent your spider from getting bad responses, +consider contacting `commercial support`_. + +.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..a7a6fd129 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,39 +226,5 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 -.. _bans: -Avoiding getting banned -======================= - -Some websites implement certain measures to prevent bots from crawling them, -with varying degrees of sophistication. Getting around those measures can be -difficult and tricky, and may sometimes require special infrastructure. Please -consider contacting `commercial support`_ if in doubt. - -Here are some tips to keep in mind when dealing with these kinds of sites: - -* rotate your user agent from a pool of well-known ones from browsers (google - around to get a list of them) -* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use - cookies to spot bot behaviour -* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites - directly -* use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ - -If you are still unable to prevent your bot getting banned, consider contacting -`commercial support`_. - -.. _Tor project: https://www.torproject.org/ -.. _commercial support: https://scrapy.org/support/ -.. _ProxyMesh: https://proxymesh.com/ -.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 572d347b3bc0149042f04ea83aff4a4f8fc7a831 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 16:17:52 +0530 Subject: [PATCH 14/34] warning view updated --- docs/topics/downloader-middleware.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 928a59bf1..99d57bda9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,8 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From cd17c829cf0d7a006ab5594d56fe182a0ffc71d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Aug 2021 19:55:35 +0500 Subject: [PATCH 15/34] Revert "Move documentation about avoiding bans into a topic of its own (#4039)" This reverts commit 2d2581c68f35799dc4372a257eaa8dbb5208481d. --- docs/index.rst | 4 - docs/topics/avoiding-bans.rst | 340 ---------------------------------- docs/topics/practices.rst | 34 ++++ 3 files changed, 34 insertions(+), 344 deletions(-) delete mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 7647b3781..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,7 +155,6 @@ Solving specific problems topics/debug topics/contracts topics/practices - topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -180,9 +179,6 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. -:doc:`topics/avoiding-bans` - Avoid getting banned from websites. - :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst deleted file mode 100644 index 59f0da191..000000000 --- a/docs/topics/avoiding-bans.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _bans: - -============= -Avoiding bans -============= - -This topic covers some of the strategies that you can follow to avoid getting -different or bad responses from a website that you are crawling due to filters -such as regional filters, web browser filters, etc. - -.. _avoiding-crawls: - -Avoiding crawls -=============== - -The best way not to be banned from a website is not to send requests to it in -the first place. - -One way to avoid crawling a website is to find the desired dataset through -other means. For example, you can use Google’s `dataset search engine`_. - -If the target website is the only or best source of the desired information, -and you only need to extract the data on a monthly basis or a lower frequency, -you may be able to crawl a public snapshot of the target website instead. -`Common Crawl`_ is an open repository of web crawl data that you can access -freely. It contains monthly snapshots of a wide variety of websites and, if you -are lucky, your target website will be among them. - -.. _Common Crawl: https://commoncrawl.org/ -.. _dataset search engine: https://datasetsearch.research.google.com/ - - -.. _being-polite: - -Being polite -============ - -To avoid being banned, you should first avoid giving a website reasons to ban -you. - -.. _identifying-yourself: - -Identifying yourself --------------------- - -If your crawling has a noticeable negative impact on a website or you crawl -content that should not be crawled, website administrators will need to do -something. - -Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and -includes contact information, so that website administrators can contact you. - - -.. _following-robotstxt: - -Following robots.txt guidelines -------------------------------- - -Some websites provide a ``robots.txt`` file at their root path (e.g. -``http://example.com/robots.txt``) that describes the guidelines that they wish -bots to follow when crawling their website. - -Before you start writing a spider for a website, read their ``robots.txt`` -file and implement your spider following its guidelines. See the `robots.txt -standard draft`_ or the `robots.txt Google specification`_ for information on -how to read ``robots.txt`` files. - -To ensure that your spider does not crawl pages restricted by ``robots.txt`` -guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the -:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` -middleware. When you do, if your spider attempts to crawl a restricted page, -this middleware aborts that request with the following message:: - - Forbidden by robots.txt - -Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and -:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or -``Request-Rate`` directives from the ``robots.txt`` guidelines. - -You may also use the :ref:`AutoThrottle extension ` on top -of that, so that when the target website experiences a high load, your spider -automatically switches to higher download delays. - -.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt -.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 - - -.. _choosing-crawl-speed: - -Finding the right guidelines on your own ----------------------------------------- - -If a website does not specify a desired download delay, or does not provide a -``robots.txt`` file, you should make an effort to find out the right values for -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that -will not have a noticeable negative impact on the target website. - -Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that -the target website receives, and choose concurrency and delay values that will -not cause a noticeable traffic increase. - -.. _SimilarWeb: https://www.similarweb.com - - -.. _filters-and-challenges: - -Bypassing filters and solving challenges -======================================== - -Some websites implement filters and challenges that aim to deny access or alter -their content based on aspects of the visitor, such as the country where they -are or the web browsing tool they use. - -.. _regional-filter: - -Bypassing regional filters --------------------------- - -Some websites send different or bad responses based on the region or country -associated to your `IP address`_. - -To bypass these filters, get access to a `proxy server`_ that has an outgoing -IP address from a region that gets the desired responses. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _IP address: https://en.wikipedia.org/wiki/IP_address -.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server - - -.. _web-browser-filter: - -Bypassing web browser filters ------------------------------ - -Some websites send different or bad responses if they detect that your request -does not come from a web browser. - -To bypass these filters, switch your :setting:`USER_AGENT` to a value copied -from those that popular web browsers use. In some rare cases, you may need a -user agent string from a specific web browser. - -There are multiple Scrapy plugins that can rotate your requests through popular -web browser user agent strings, such as scrapy-fake-useragent_, -scrapy-random-useragent_ or Scrapy-UserAgents_. - -For advanced web browser filters, -:ref:`pre-rendering JavaScript ` or -:ref:`using a headless browser ` may be necessary. -Use these options only as a last resort, however, because they cause a higher -load per request on the target website. - -.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent -.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent -.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ - - -.. _request-delay-filter: - -Bypassing request delay filters -------------------------------- - -Some websites may ban your IP after they detect that your requests use a -constant download delay. - -To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting is enabled by default. When that is not enough, an -:ref:`IP address rotation solution ` may be much more effective. - - -.. _isp-filter: - -Bypassing internet service provider filters -------------------------------------------- - -Some websites send different or bad responses if they detect that your request -comes from an IP address that belongs to a `data center`_, as opposed to a -residential IP address from an `internet service provider`_ or a mobile IP -address from a `mobile network`_. - -To bypass these filters, get access to a proxy server that has an outgoing IP -address that is either residential or mobile. Note that you may also get -different responses depending on whether your IP address is residential or -mobile. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _data center: https://en.wikipedia.org/wiki/Data_center -.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider -.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network - - -.. _captcha: - -Solving CAPTCHA challenges --------------------------- - -Some websites require you to solve a `CAPTCHA challenge`_ to get the desired -response. - -To bypass these filters, several options exist: - -- You could have your spider present the CAPTCHA challenge to you and wait - for you to solve it manually. - -- Some CAPTCHA challenges can be solved using an `optical character - recognition`_ (OCR) solution such as pytesseract_. - -- Paid CAPTCHA solving services exist. - -Whichever solution you choose, implement it as a :ref:`downloader middleware -` that automatically detects CAPTCHA challenges -in responses and solves them, so that your spider code only receives successful -responses. - -.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA -.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition -.. _pytesseract: https://github.com/madmaze/pytesseract - - -.. _ip-rotation: - -IP address rotation solutions -============================= - -See below some of the different solutions there are to have your requests use -different outgoing IP addresses. - -When using this approach, remember to set :setting:`COOKIES_ENABLED` to -``False`` to disable global cookie handling. This prevents websites from -identifying two requests as coming from the same user agent even if they come -from different IP addresses and have different user-agent strings. You can -still include some cookies manually in your requests. Define them through the -``Cookies`` header of your requests. See -:class:`Request.headers `. - -.. _smart-proxy: - -Smart proxies -------------- - -An increasing number of websites use solutions that apply many of the above -filters and challenges at the same time. - -There are paid proxy services, like `Zyte Smart Proxy Manager`_, that -automatically bypass website filters and challenges, so that your spider only -gets successful responses. They also allow managing sessions to simulate user -behavior. - -For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced -integration with Scrapy. For other services, use the -:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware -or implement your own :ref:`downloader middleware -`. - -.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ - - -.. _rotating-proxy: - -Rotating proxies ----------------- - -Rotating proxy services like ProxyMesh_ send different requests through -different proxies. This can decrease the likelihood of being affected by some -filters or challenges. - -.. _ProxyMesh: https://proxymesh.com/ - - -.. _free-proxies: - -Free proxies ------------- - -You can easily find lists of free proxies in the internet, and you can use -a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in -your spider and have requests rotate through them automatically. - -This approach, however, has serious drawbacks: - -- Free proxies may stop working at any moment. You need to implement a way to - refresh your list of free proxies. - -- In addition to handling occasional bad responses from websites, you - need to handle all kinds of bad responses from proxies. You may even need - to inspect the response body to determine if a response comes from the - target website or from a misbehaving proxy. - -- Advanced antibot solutions may automatically detect and filter out traffic - from free proxies. - -.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies - - -.. _custom-rotating-proxy: - -Custom rotating proxy ---------------------- - -If you have spare servers, you can set them up as proxies and use scrapoxy_ to -build a custom proxy that rotates traffic through them. However, the initial -setup can be complex, and your requests will be vulnerable to -:ref:`internet service provider filtering `. - -.. _scrapoxy: https://scrapoxy.io/ - - -.. _tor: - -The Tor network ---------------- - -It is possible to send requests through the `Tor network`_. - -The initial setup to have Scrapy working with Tor is not straightforward. -Use a search engine to find up-to-date documentation specific to using -Scrapy and Tor together. - -The main drawback of using the Tor network is that traffic can be extremely -slow. - -.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) - - -.. _commercial-support: - -Seeking professional help -========================= - -Avoiding bans, filters and challenges can be difficult and tricky, and may -sometimes require special infrastructure. - -If you find yourself unable to prevent your spider from getting bad responses, -consider contacting `commercial support`_. - -.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a7a6fd129..732eba587 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,5 +226,39 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 +.. _bans: +Avoiding getting banned +======================= + +Some websites implement certain measures to prevent bots from crawling them, +with varying degrees of sophistication. Getting around those measures can be +difficult and tricky, and may sometimes require special infrastructure. Please +consider contacting `commercial support`_ if in doubt. + +Here are some tips to keep in mind when dealing with these kinds of sites: + +* rotate your user agent from a pool of well-known ones from browsers (google + around to get a list of them) +* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use + cookies to spot bot behaviour +* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. +* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites + directly +* use a pool of rotating IPs. For example, the free `Tor project`_ or paid + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a + super proxy that you can attach your own proxies to. +* use a highly distributed downloader that circumvents bans internally, so you + can just focus on parsing clean pages. One example of such downloaders is + `Zyte Smart Proxy Manager`_ + +If you are still unable to prevent your bot getting banned, consider contacting +`commercial support`_. + +.. _Tor project: https://www.torproject.org/ +.. _commercial support: https://scrapy.org/support/ +.. _ProxyMesh: https://proxymesh.com/ +.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders +.. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 3f635eb683821667b7a46a99531a95a8c05b6e1b Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Tue, 24 Aug 2021 12:05:50 +0300 Subject: [PATCH 16/34] Extract domain from genspider URL (#4439) --- scrapy/commands/genspider.py | 12 +++++++++++- tests/test_commands.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5f44daa70..2082a4974 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, dirname, abspath, exists, splitext +from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand @@ -22,6 +23,14 @@ def sanitize_module_name(module_name): return module_name +def extract_domain(url): + """Extract domain name from URL string""" + o = urlparse(url) + if o.scheme == '' and o.netloc == '': + o = urlparse("//" + url.lstrip("/")) + return o.netloc + + class Command(ScrapyCommand): requires_project = False @@ -59,7 +68,8 @@ class Command(ScrapyCommand): if len(args) != 2: raise UsageError() - name, domain = args[0:2] + name, url = args[0:2] + domain = extract_domain(url) module = sanitize_module_name(name) if self.settings.get('BOT_NAME') == module: diff --git a/tests/test_commands.py b/tests/test_commands.py index 74b917d93..086286b3a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import json import optparse import os import platform +import re import subprocess import sys import tempfile @@ -94,6 +95,15 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) + def find_in_file(self, filename, regex): + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with open(filename, "r") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + class StartprojectTest(ProjectTest): @@ -482,6 +492,26 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) + def test_url(self, url='test.com', domain="test.com"): + self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) + self.assertEqual(domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) + self.assertEqual('http://%s/' % domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + + def test_url_schema(self): + self.test_url('http://test.com', 'test.com') + + def test_url_path(self): + self.test_url('test.com/some/other/page', 'test.com') + + def test_url_schema_path(self): + self.test_url('https://test.com/some/other/page', 'test.com') + class GenspiderStandaloneCommandTest(ProjectTest): From 43ea21e8306bc66e8f07abc84cce680726abc7dc Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 24 Aug 2021 15:18:01 +0530 Subject: [PATCH 17/34] Feed post-processing plugin support (#5190) --- .github/workflows/tests-ubuntu.yml | 2 +- docs/topics/feed-exports.rst | 67 +++- scrapy/extensions/feedexport.py | 4 + scrapy/extensions/postprocessing.py | 154 +++++++++ tests/test_feedexport.py | 496 ++++++++++++++++++++++++++++ 5 files changed, 721 insertions(+), 2 deletions(-) create mode 100644 scrapy/extensions/postprocessing.py diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 81beda5da..ef1c8362f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,7 +20,7 @@ jobs: - python-version: pypy3 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 + PYPY_VERSION: 3.6-v7.3.3 # pinned deps - python-version: 3.6.12 diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2b3217d62..116967280 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. + .. _item-filter: Item filtering @@ -312,6 +313,63 @@ ItemFilter :members: +.. _post-processing: + +Post-Processing +=============== + +.. versionadded:: VERSION + +Scrapy provides an option to activate plugins to post-process feeds before they are exported +to feed storages. In addition to using :ref:`builtin plugins `, you +can create your own :ref:`plugins `. + +These plugins can be activated through the ``postprocessing`` option of a feed. +The option must be passed a list of post-processing plugins in the order you want +the feed to be processed. These plugins can be declared either as an import string +or with the imported class of the plugin. Parameters to plugins can be passed +through the feed options. See :ref:`feed options ` for examples. + +.. _builtin-plugins: + +Built-in Plugins +---------------- + +.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin + +.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin + +.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin + +.. _custom-plugins: + +Custom Plugins +-------------- + +Each plugin is a class that must implement the following methods: + +.. method:: __init__(self, file, feed_options) + + Initialize the plugin. + + :param file: file-like object having at least the `write`, `tell` and `close` methods implemented + + :param feed_options: feed-specific :ref:`options ` + :type feed_options: :class:`dict` + +.. method:: write(self, data) + + Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file. + It must return number of bytes written. + +.. method:: close(self) + + Close the target file object. + +To pass a parameter to your plugin, use :ref:`feed options `. You +can then access those parameters from the ``__init__`` method of your plugin. + + Settings ======== @@ -368,10 +426,12 @@ For instance:: 'encoding': 'latin1', 'indent': 8, }, - pathlib.Path('items.csv'): { + pathlib.Path('items.csv.gz'): { 'format': 'csv', 'fields': ['price', 'name'], 'item_filter': 'myproject.filters.MyCustomFilter2', + 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 5, }, } @@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition: - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. +- ``postprocessing``: list of :ref:`plugins ` to use for post-processing. + + The plugins will be used in the order of the list passed. + + .. versionadded:: VERSION .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 564c736f2..0f5bf01d0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -20,6 +20,7 @@ from zope.interface import implementer, Interface from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file @@ -396,6 +397,9 @@ class FeedExporter: """ storage = self._get_storage(uri, feed_options) file = storage.open(spider) + if "postprocessing" in feed_options: + file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + exporter = self._get_exporter( file=file, format=feed_options['format'], diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py new file mode 100644 index 000000000..413c2e55e --- /dev/null +++ b/scrapy/extensions/postprocessing.py @@ -0,0 +1,154 @@ +""" +Extension for processing data before they are exported to feeds. +""" +from bz2 import BZ2File +from gzip import GzipFile +from io import IOBase +from lzma import LZMAFile +from typing import Any, BinaryIO, Dict, List + +from scrapy.utils.misc import load_object + + +class GzipPlugin: + """ + Compresses received data using `gzip `_. + + Accepted ``feed_options`` parameters: + + - `gzip_compresslevel` + - `gzip_mtime` + - `gzip_filename` + + See :py:class:`gzip.GzipFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("gzip_compresslevel", 9) + mtime = self.feed_options.get("gzip_mtime") + filename = self.feed_options.get("gzip_filename") + self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, + mtime=mtime, filename=filename) + + def write(self, data: bytes) -> int: + return self.gzipfile.write(data) + + def close(self) -> None: + self.gzipfile.close() + self.file.close() + + +class Bz2Plugin: + """ + Compresses received data using `bz2 `_. + + Accepted ``feed_options`` parameters: + + - `bz2_compresslevel` + + See :py:class:`bz2.BZ2File` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("bz2_compresslevel", 9) + self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + + def write(self, data: bytes) -> int: + return self.bz2file.write(data) + + def close(self) -> None: + self.bz2file.close() + self.file.close() + + +class LZMAPlugin: + """ + Compresses received data using `lzma `_. + + Accepted ``feed_options`` parameters: + + - `lzma_format` + - `lzma_check` + - `lzma_preset` + - `lzma_filters` + + .. note:: + ``lzma_filters`` cannot be used in pypy version 7.3.1 and older. + + See :py:class:`lzma.LZMAFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + + format = self.feed_options.get("lzma_format") + check = self.feed_options.get("lzma_check", -1) + preset = self.feed_options.get("lzma_preset") + filters = self.feed_options.get("lzma_filters") + self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, + check=check, preset=preset, filters=filters) + + def write(self, data: bytes) -> int: + return self.lzmafile.write(data) + + def close(self) -> None: + self.lzmafile.close() + self.file.close() + + +# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager +# instance as a file like writable object. This could be needed by some exporters +# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper. +class PostProcessingManager(IOBase): + """ + This will manage and use declared plugins to process data in a + pipeline-ish way. + :param plugins: all the declared plugins for the feed + :type plugins: list + :param file: final target file where the processed data will be written + :type file: file like object + """ + + def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.plugins = self._load_plugins(plugins) + self.file = file + self.feed_options = feed_options + self.head_plugin = self._get_head_plugin() + + def write(self, data: bytes) -> int: + """ + Uses all the declared plugins to process data first, then writes + the processed data to target file. + :param data: data passed to be written to target file + :type data: bytes + :return: returns number of bytes written + :rtype: int + """ + return self.head_plugin.write(data) + + def tell(self) -> int: + return self.file.tell() + + def close(self) -> None: + """ + Close the target file along with all the plugins. + """ + self.head_plugin.close() + + def writable(self) -> bool: + return True + + def _load_plugins(self, plugins: List[Any]) -> List[Any]: + plugins = [load_object(plugin) for plugin in plugins] + return plugins + + def _get_head_plugin(self) -> Any: + prev = self.file + for plugin in self.plugins[::-1]: + prev = plugin(prev, self.feed_options) + return prev diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 53e6a2018..253f3119c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,5 +1,8 @@ +import bz2 import csv +import gzip import json +import lzma import os import random import shutil @@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(row['expected'], data[feed_options['format']]) +class FeedPostProcessedExportsTest(FeedExportTestBase): + __test__ = True + + items = [{'foo': 'bar'}] + expected = b'foo\r\nbar\r\n' + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get('plugin1_char', b'') + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name): + return os.path.join(self.temp_dir, name) + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data with filename. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[str(file_path)] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + data_stream = BytesIO() + gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, + compresslevel=compresslevel, mode="wb") + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @defer.inlineCallbacks + def test_gzip_plugin(self): + + filename = self._named_tempfile('gzip_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + self.fail("Received invalid gzip data.") + + @defer.inlineCallbacks + def test_gzip_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), + self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 0, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 9, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), + self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + } + + settings = { + 'FEEDS': { + self._named_tempfile('mtime_123'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123, + 'gzip_filename': "", + }, + self._named_tempfile('mtime_123456789'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123456789, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), + self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + } + + settings = { + 'FEEDS': { + self._named_tempfile('filename_FILE1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE1", + }, + self._named_tempfile('filename_FILE2'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE2", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin(self): + + filename = self._named_tempfile('lzma_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + self.fail("Received invalid lzma data.") + + @defer.inlineCallbacks + def test_lzma_plugin_format(self): + + filename_to_compressed = { + self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), + self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + } + + settings = { + 'FEEDS': { + self._named_tempfile('format_FORMAT_XZ'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_XZ, + }, + self._named_tempfile('format_FORMAT_ALONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_ALONE, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_check(self): + + filename_to_compressed = { + self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), + self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + } + + settings = { + 'FEEDS': { + self._named_tempfile('check_CHECK_NONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_NONE, + }, + self._named_tempfile('check_CHECK_CRC256'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_SHA256, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_preset(self): + + filename_to_compressed = { + self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), + self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('preset_PRESET_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 0, + }, + self._named_tempfile('preset_PRESET_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_filters(self): + import sys + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + raise unittest.SkipTest("lzma filters doesn't work in PyPy") + + filters = [{'id': lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile('filters') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_filters': filters, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(compressed, data[filename]) + result = lzma.decompress(data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_bz2_plugin(self): + + filename = self._named_tempfile('bz2_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + self.fail("Received invalid bz2 data.") + + @defer.inlineCallbacks + def test_bz2_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), + self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 1, + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_custom_plugin(self): + filename = self._named_tempfile('csv_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(self.expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_parameter(self): + + expected = b'foo\r\n\nbar\r\n\n' + filename = self._named_tempfile('newline') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + 'plugin1_char': b'\n' + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_compression(self): + + expected = b'foo\r\n\nbar\r\n\n' + + filename_to_decompressor = { + self._named_tempfile('bz2'): bz2.decompress, + self._named_tempfile('lzma'): lzma.decompress, + self._named_tempfile('gzip'): gzip.decompress, + } + + settings = { + 'FEEDS': { + self._named_tempfile('bz2'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('lzma'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('gzip'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'plugin1_char': b'\n', + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def test_exports_compatibility_with_postproc(self): + import marshal + import pickle + filename_to_expected = { + self._named_tempfile('csv'): b'foo\r\nbar\r\n', + self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', + self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', + self._named_tempfile('xml'): b'\n' + b'\nbar\n', + } + + settings = { + 'FEEDS': { + self._named_tempfile('csv'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile('json'): { + 'format': 'json', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('jsonlines'): { + 'format': 'jsonlines', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('xml'): { + 'format': 'xml', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('marshal'): { + 'format': 'marshal', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('pickle'): { + 'format': 'pickle', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, result in data.items(): + if 'pickle' in filename: + expected, result = self.items[0], pickle.loads(result) + elif 'marshal' in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + self.assertEqual(expected, result) + + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)02d_' From 8284de5e7613c47e69d40d4f6a2a1dc846b50dd6 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 24 Aug 2021 15:15:29 +0500 Subject: [PATCH 18/34] Fix/silence the Pylint messages added in 2.10 (#5235) --- pylintrc | 1 + scrapy/utils/conf.py | 2 +- scrapy/utils/deprecate.py | 2 +- tests/keys/__init__.py | 8 ++++---- tests/test_downloader_handlers_http2.py | 2 +- tests/test_engine.py | 2 +- tests/test_exporters.py | 4 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler.py | 10 +++++----- tests/test_utils_conf.py | 4 ++-- tests/test_utils_misc/__init__.py | 2 +- 13 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pylintrc b/pylintrc index a44712507..699686e16 100644 --- a/pylintrc +++ b/pylintrc @@ -105,6 +105,7 @@ disable=abstract-method, unnecessary-lambda, unnecessary-pass, unreachable, + unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index b904c4a03..24873f75d 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) - out.setdefault("item_export_kwargs", dict()) + out.setdefault("item_export_kwargs", {}) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f5b17416f..ae727464c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -79,7 +79,7 @@ def create_deprecated_class( # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) - for c in {type(inst), inst.__class__}) + for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index da202be4d..bb4a8e5af 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -40,9 +40,9 @@ def generate_keys(): subject = issuer = Name( [ - NameAttribute(NameOID.COUNTRY_NAME, u"IE"), - NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), - NameAttribute(NameOID.COMMON_NAME, u"localhost"), + NameAttribute(NameOID.COUNTRY_NAME, "IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"), + NameAttribute(NameOID.COMMON_NAME, "localhost"), ] ) cert = ( @@ -54,7 +54,7 @@ def generate_keys(): .not_valid_before(datetime.utcnow()) .not_valid_after(datetime.utcnow() + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName(u"localhost")]), + SubjectAlternativeName([DNSName("localhost")]), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 53bb4fe92..8c8c30597 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): certfile = 'keys/localhost.crt' scheme = 'https' - host = u'127.0.0.1' + host = '127.0.0.1' expected_http_proxy_request_body = b'/' diff --git a/tests/test_engine.py b/tests/test_engine.py index 92bf45f25..fa7d0c8d4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -152,7 +152,7 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.headers = {} - self.bytes = defaultdict(lambda: list()) + self.bytes = defaultdict(list) self.signals_caught = {} self.spider_class = spider_class diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 04bae31d3..b263b3475 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), expected=None, encoding='windows-1251', ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), include_headers_line=False, expected='Wɵ​rd\r\n', encoding='windows-1251', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 677ede92b..49c83132f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = u'localhost' + self.hostname = 'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 41afa2896..0fd52da5f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): return None with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context=dict()) + wrap_loader_context(function, context={}) assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b68184b87..738502de8 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -57,7 +57,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = list() + checks = [] def start_requests(self): data = {'key': 'value', 'number': 123} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 512a7460e..2d4bfa165 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active']) class MockDownloader: def __init__(self): - self.slots = dict() + self.slots = {} def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: @@ -31,7 +31,7 @@ class MockDownloader: return urlparse_cached(request).hostname or '' def increment(self, slot_key): - slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) def decrement(self, slot_key): @@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): for url, priority in _PRIORITIES: self.scheduler.enqueue_request(Request(url, priority=priority)) - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin: self.close_scheduler() self.create_scheduler() - dequeued_slots = list() + dequeued_slots = [] requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc2560add..a92880626 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 47d73a2dd..b83c1d6f0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - self.assertRaises(TypeError, load_object, dict()) + self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From ac9175964dda07da1e838ebb063fc8dd95925e0d Mon Sep 17 00:00:00 2001 From: maanijou <19888963+maanijou@users.noreply.github.com> Date: Sun, 12 Sep 2021 17:59:20 +0200 Subject: [PATCH 19/34] Improve documentation for spider middlewares --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f0158dc41..f3fb0d5d7 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, From e5998fb8469dff1e2e965826d2b43f9bad0c17ad Mon Sep 17 00:00:00 2001 From: kamran890 Date: Wed, 22 Sep 2021 03:00:18 +0500 Subject: [PATCH 20/34] Document spider.state attribute (#5174) --- docs/topics/jobs.rst | 2 ++ docs/topics/spiders.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index e49f37a2f..f16d306c7 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 +.. _topics-keeping-persistent-state-between-batches: + Keeping persistent state between batches ======================================== diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 67b9e2e0e..4d3d32941 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -122,6 +122,11 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. + .. attribute:: state + + A dict you can use to persist some spider state between batches. + See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. From 1829dd774ca0f056e97f7c4621575ef9126e4b57 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sat, 25 Sep 2021 20:22:09 +0330 Subject: [PATCH 21/34] Update spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f3fb0d5d7..08be2b03c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From dfdb779756aa1df2059980de02d359e4e93bac55 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sun, 26 Sep 2021 12:45:44 +0330 Subject: [PATCH 22/34] Apply review comments --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 08be2b03c..f1373b9ee 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item objects + iterable of :class:`~scrapy.Request` or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From 3c57825b0f3a7bb250aec753b09f964f36500e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 26 Sep 2021 13:41:26 +0200 Subject: [PATCH 23/34] Update docs/topics/spider-middleware.rst --- docs/topics/spider-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f1373b9ee..73bedf655 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` or :ref:`item objects - `. + iterable of :class:`~scrapy.Request` or :ref:`item ` + objects. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following From 890f884de46602352de48fa844edd9959c62e473 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 1 Oct 2021 04:50:42 -0300 Subject: [PATCH 24/34] Allow 'callback' key in keyword arguments for request callbacks (#5251) --- scrapy/core/scraper.py | 2 +- tests/test_request_cb_kwargs.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d6d6f64f9..f40bccbb3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -156,7 +156,7 @@ class Scraper: callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallback(callback, **result.request.cb_kwargs) + dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 738502de8..8b96fe1a1 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -60,7 +60,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123} + data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) @@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider): if response.url.endswith('/general_with'): self.checks.append(kwargs['key'] == 'value') self.checks.append(kwargs['number'] == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.checks.append(kwargs['callback'] == 'some_callback') + self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): self.checks.append(kwargs == {}) self.crawler.stats.inc_value('boolean_checks') @@ -110,7 +111,7 @@ class KeywordArgumentsSpider(MockServerSpider): TypeError: parse_takes_less() got an unexpected keyword argument 'number' """ - def parse_takes_more(self, response, key, number, other): + def parse_takes_more(self, response, key, number, callback, other): """ Should raise TypeError: parse_takes_more() missing 1 required positional argument: 'other' @@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertTrue( str(exceptions['takes_less'].exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ) + ), + msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) self.assertTrue( str(exceptions['takes_more'].exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ) + ), + msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), ) From d91d82b5064c512e741da106b5fb3a398027d5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Oct 2021 16:31:29 +0200 Subject: [PATCH 25/34] Make Scrapy SFW again --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d3e08efd4..42ce22158 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -300,7 +300,7 @@ errors if needed:: "http://www.httpbin.org/status/404", # Not found error "http://www.httpbin.org/status/500", # server issue "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "http://www.httphttpbinbin.org/", # DNS error expected + "https://example.invalid/", # DNS error expected ] def start_requests(self): From ef263042d75586e94d74290d1a41c432f7d87bec Mon Sep 17 00:00:00 2001 From: Raihan Nismara <31585789+raihan71@users.noreply.github.com> Date: Sun, 3 Oct 2021 13:26:20 +0700 Subject: [PATCH 26/34] Using Logo Scrapy in Readme.md Logo scrapy used in readme.md made looks nicer --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 5750e2c0f..05f10bb6c 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: /artwork/scrapy-logo.jpg + :width: 400px + ====== Scrapy ====== From b081f18a2f232a1bfd04c829ae6894cac93a89a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 16 Aug 2019 14:53:42 +0500 Subject: [PATCH 27/34] Add http_auth_domain to HttpAuthMiddleware. --- docs/topics/downloader-middleware.rst | 18 ++++- scrapy/downloadermiddlewares/httpauth.py | 21 ++++- tests/test_downloadermiddleware_httpauth.py | 87 ++++++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 99d57bda9..28b019c80 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -323,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but usually this is not needed. + + .. warning:: + In the previous Scrapy versions HttpAuthMiddleware sent the + authentication data with all requests, which is a security problem if + the spider makes requests to several different domains. Currently if + the ``http_auth_domain`` attribute is not set, the middleware will use + the domain of the first request, which will work for some spider but + not for others. In the future the middleware will produce an error + instead. Example:: @@ -334,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..1bee3e279 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,10 +3,14 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +import warnings from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.url import url_is_from_any_domain class HttpAuthMiddleware: @@ -24,8 +28,23 @@ class HttpAuthMiddleware: pwd = getattr(spider, 'http_pass', '') if usr or pwd: self.auth = basic_auth_header(usr, pwd) + if not hasattr(spider, 'http_auth_domain'): + warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' + 'problems if the spider makes requests to several different domains. http_auth_domain ' + 'will be set to the domain of the first request, please set it to the correct value ' + 'explicitly.', + category=ScrapyDeprecationWarning) + self.domain_unset = True + else: + self.domain = spider.http_auth_domain + self.domain_unset = False def process_request(self, request, spider): auth = getattr(self, 'auth', None) if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth + domain = urlparse_cached(request).hostname + if self.domain_unset: + self.domain = domain + self.domain_unset = False + if not self.domain or url_is_from_any_domain(request.url, [self.domain]): + request.headers[b'Authorization'] = auth diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..0362e2018 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,13 +1,60 @@ import unittest +from w3lib.http import basic_auth_header + from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider +class TestSpiderLegacy(Spider): + http_user = 'foo' + http_pass = 'bar' + + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + http_auth_domain = 'example.com' + + +class TestSpiderAny(Spider): + http_user = 'foo' + http_pass = 'bar' + http_auth_domain = None + + +class HttpAuthMiddlewareLegacyTest(unittest.TestCase): + + def setUp(self): + self.spider = TestSpiderLegacy('foo') + + def test_auth(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + + # initial request, sets the domain and sends the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to the same domain, should send the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to a different domain, shouldn't send the header + req = Request('http://example-noauth.com/') + assert mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_auth_already_set(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') class HttpAuthMiddlewareTest(unittest.TestCase): @@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw - def test_auth(self): - req = Request('http://scrapytest.org/') + def test_no_auth(self): + req = Request('http://example-noauth.com/') assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertNotIn('Authorization', req.headers) + + def test_auth_domain(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_subdomain(self): + req = Request('http://foo.example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) def test_auth_already_set(self): - req = Request('http://scrapytest.org/', + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + +class HttpAuthAnyMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpiderAny('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_already_set(self): + req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') From 7ec5f299c42d07768c02970f0a11f018ed790188 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 22 Aug 2019 20:32:56 +0500 Subject: [PATCH 28/34] Small documentation fixes. --- docs/topics/downloader-middleware.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 28b019c80..caf44a903 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -328,16 +328,16 @@ HttpAuthMiddleware ``http_auth_domain`` spider attribute to the domain which requires this authentication (its subdomains will be also handled in the same way). You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but usually this is not needed. + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. .. warning:: - In the previous Scrapy versions HttpAuthMiddleware sent the - authentication data with all requests, which is a security problem if - the spider makes requests to several different domains. Currently if - the ``http_auth_domain`` attribute is not set, the middleware will use - the domain of the first request, which will work for some spider but - not for others. In the future the middleware will produce an error - instead. + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: From f0105a882df200f71088603fecaeb9d40679c387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 13:29:06 +0200 Subject: [PATCH 29/34] Cover 2.5.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0ea412e75..4b5cbb2da 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,44 @@ Release notes ============= +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-2.5.0: Scrapy 2.5.0 (2021-04-06) From 735750c254e6e82af46b4ebbb35e28b8c0a52250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 21:10:49 +0200 Subject: [PATCH 30/34] Cover 1.8.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 4b5cbb2da..5e590f027 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1492,6 +1492,44 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-1.8.0: Scrapy 1.8.0 (2019-10-28) From 6c858cec91b013853a73e6215b74c90b609bc2da Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Oct 2021 12:32:04 -0300 Subject: [PATCH 31/34] Cookies: Cast primitive types to str (#5253) * cast primitive types to str * add tests --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- tests/test_downloadermiddleware_cookies.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d95ed3d38..0eee8d758 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -80,8 +80,8 @@ class CookiesMiddleware: logger.warning(msg.format(request, cookie, key)) return continue - if isinstance(cookie[key], str): - decoded[key] = cookie[key] + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) else: try: decoded[key] = cookie[key].decode("utf8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index aff8542e9..36021bfbf 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -347,3 +347,24 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + + def test_primitive_type_cookies(self): + # Boolean + req1 = Request('http://example.org', cookies={'a': True}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + + # Float + req2 = Request('http://example.org', cookies={'a': 9.5}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + + # Integer + req3 = Request('http://example.org', cookies={'a': 10}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + + # String + req4 = Request('http://example.org', cookies={'a': 'b'}) + assert self.mw.process_request(req4, self.spider) is None + self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') From 029cab72e8a9b20ebb6b9540e9e01747f0e83dba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 6 Oct 2021 14:34:09 -0300 Subject: [PATCH 32/34] [CI] fix pypy test (#5264) --- tests/test_request_cb_kwargs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8b96fe1a1..473a93e69 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -105,7 +105,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(default == 99) self.crawler.stats.inc_value('boolean_checks', 4) - def parse_takes_less(self, response, key): + def parse_takes_less(self, response, key, callback): """ Should raise TypeError: parse_takes_less() got an unexpected keyword argument 'number' From d3f1bf79e883fe3662df827ee47cfc93a372ff02 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 7 Oct 2021 17:27:20 +0300 Subject: [PATCH 33/34] Use f-strings where appropriate (#5246) --- scrapy/__init__.py | 2 +- scrapy/core/downloader/contextfactory.py | 12 +++++---- scrapy/core/downloader/tls.py | 8 +++--- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/extensions/feedexport.py | 33 ++++++++++------------- scrapy/extensions/httpcache.py | 4 +-- scrapy/utils/misc.py | 2 +- tests/spiders.py | 16 +++++------ tests/test_closespider.py | 2 +- tests/test_commands.py | 4 +-- tests/test_downloader_handlers_http2.py | 2 +- tests/test_http_request.py | 19 +++++++------ 12 files changed, 51 insertions(+), 55 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 8a8065bf2..396f98219 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version if sys.version_info < (3, 6): - print("Scrapy %s requires Python 3.6+" % __version__) + print(f"Scrapy {__version__} requires Python 3.6+") sys.exit(1) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 073ef16bf..b5318c7bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler): settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = ( + f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " + "a `method` argument (type OpenSSL.SSL method, e.g. " + "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " + "argument and/or a `tls_ciphers` argument. Please, upgrade your " + "context factory class to handle them or ignore them." + ) warnings.warn(msg) return context_factory diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 2b8990b75..19a56d9b6 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions): verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Remote certificate is not valid for hostname "%s"; %s', + self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + 'from host "%s" (exception: %r)', + self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 62f1c3a29..80ed7ac75 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -70,7 +70,7 @@ class HttpCacheMiddleware: self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) - raise IgnoreRequest("Ignored request not in cache: %s" % request) + raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0f5bf01d0..370723368 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): kwargs['feed_options'] = feed_options else: warnings.warn( - "{} does not support the 'feed_options' keyword argument. Add a " + f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " - "version of Scrapy." - .format(builder.__qualname__), + "version of Scrapy.", category=ScrapyDeprecationWarning ) return builder(*preargs, uri, *args, **kwargs) @@ -356,32 +355,28 @@ class FeedExporter: # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope d.addCallback( - self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_success, logmsg, spider, type(slot.storage).__name__ ) d.addErrback( - self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) return d - def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - logfmt % "Error storing", largs, + "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + def _handle_store_success(self, f, logmsg, spider, slot_type): logger.info( - logfmt % "Stored", largs, extra={'spider': spider} + "Stored %s", logmsg, + extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") @@ -474,10 +469,10 @@ class FeedExporter: for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' - ''.format(uri_template) + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', + uri_template ) return False return True @@ -526,7 +521,7 @@ class FeedExporter: instance = build_instance(feedcls) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance def _get_uri_params(self, spider, uri_params, slot=None): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index e0c04b2de..d0ae29b90 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -226,7 +226,7 @@ class DbmCacheStorage: dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') - logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) def close_spider(self, spider): self.db.close() @@ -280,7 +280,7 @@ class FilesystemCacheStorage: self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) def close_spider(self, spider): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 51cef1e91..11c4206c2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -50,7 +50,7 @@ def load_object(path): return path else: raise TypeError("Unexpected argument type, expected string " - "or object, got: %s" % type(path)) + f"or object, got: {type(path)}") try: dot = path.rindex('.') diff --git a/tests/spiders.py b/tests/spiders.py index 5b45f897e..67dbbbe0f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider): self.start_urls = [url] def parse(self, response): - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): @@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider): async def parse(self, response): await defer.succeed(42) - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioSpider(SimpleSpider): @@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") class AsyncDefAsyncioReturnSpider(SimpleSpider): @@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return [{'id': 1}, {'id': 2}] @@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return {"foo": 42} @@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): await asyncio.sleep(0.2) req_id = response.meta.get('req_id', 0) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d, req_id %d" % (status, req_id)) + self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: return reqs = [] @@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) yield {'foo': 42} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): @@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): for i in range(10): await asyncio.sleep(0.1) yield {'foo': i} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 5ec5e2989..be8adadb3 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,7 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) + key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_commands.py b/tests/test_commands.py index 086286b3a..75098a77a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest): self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) - self.assertEqual('http://%s/' % domain, + self.assertEqual(f'http://{domain}/', self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) @@ -708,7 +708,7 @@ class MySpider(scrapy.Spider): ]) import asyncio loop = asyncio.new_event_loop() - self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): spider_code = """ diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8c8c30597..3a9db3ee5 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'/') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f"{self.getURL('')}?noconnect" request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex( Warning, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b610087bd..579ef9fa2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest): response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): - body = """
- -
""" + form_methods = [[method, method] for method in self.request_class.valid_form_methods] + form_methods.append(['UNKNOWN', 'GET']) - for method in self.request_class.valid_form_methods: - response = _buildresponse(body % method) + for method, expected in form_methods: + response = _buildresponse( + f'
' + '' + '
' + ) r = self.request_class.from_response(response) - self.assertEqual(r.method, method) - - response = _buildresponse(body % 'UNKNOWN') - r = self.request_class.from_response(response) - self.assertEqual(r.method, 'GET') + self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): From 65d60b9692dc3475b42d14c744d8a5ac0f2b38cf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sun, 10 Oct 2021 05:06:36 -0300 Subject: [PATCH 34/34] [docs] add missing parameter to headers_received signal (#5270) --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a67cc1879..63ad3a9ad 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -413,7 +413,7 @@ headers_received .. versionadded:: 2.5 .. signal:: headers_received -.. function:: headers_received(headers, request, spider) +.. function:: headers_received(headers, body_length, request, spider) Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content.