From 9d32255e86ddc620fc099bbd72c8c4ea9c2a48c5 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Sat, 29 Aug 2015 13:28:40 -0400 Subject: [PATCH 01/29] parse authentication credentials from url for http and ftp schemes --- scrapy/downloadermiddlewares/auth.py | 46 +++++++++++++++++++ scrapy/downloadermiddlewares/httpauth.py | 36 +++------------ scrapy/settings/default_settings.py | 2 +- ...h.py => test_downloadermiddleware_auth.py} | 6 +-- 4 files changed, 56 insertions(+), 34 deletions(-) create mode 100644 scrapy/downloadermiddlewares/auth.py rename tests/{test_downloadermiddleware_httpauth.py => test_downloadermiddleware_auth.py} (83%) diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py new file mode 100644 index 000000000..b832d569a --- /dev/null +++ b/scrapy/downloadermiddlewares/auth.py @@ -0,0 +1,46 @@ +""" +HTTP basic auth downloader middleware + +See documentation in docs/topics/downloader-middleware.rst +""" + +from w3lib.http import basic_auth_header + +from scrapy import signals + +from six.moves.urllib.parse import urlparse + + +class AuthMiddleware(object): + """Set Basic HTTP Authorization header + (http_user and http_pass spider class attributes)""" + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o + + def spider_opened(self, spider): + usr = getattr(spider, 'http_user', '') + pwd = getattr(spider, 'http_pass', '') + if usr or pwd: + self.auth = basic_auth_header(usr, pwd) + + def process_request(self, request, spider): + auth = getattr(self, 'auth', None) + if auth and 'Authorization' not in request.headers: + request.headers['Authorization'] = auth + + # credentials from url are supposed to override spider settings + url = urlparse(request.url) + if url.username and url.password: + if url.scheme.startswith('ftp'): + request.meta['ftp_user'] = url.username + request.meta['ftp_password'] = url.password + elif url.scheme.startswith('http'): + request.headers['Authorization'] = basic_auth_header(url.username, url.password) + + # no credentials in new url + new_url = url.scheme + '://' + url.hostname + url.path + return request.replace(url=new_url) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7aa7a62bc..78adc8706 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,31 +1,7 @@ -""" -HTTP basic auth downloader middleware +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, " + "use `scrapy.downloadermiddlewares.auth` instead", + ScrapyDeprecationWarning) -See documentation in docs/topics/downloader-middleware.rst -""" - -from w3lib.http import basic_auth_header - -from scrapy import signals - - -class HttpAuthMiddleware(object): - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" - - @classmethod - def from_crawler(cls, crawler): - o = cls() - crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) - return o - - def spider_opened(self, spider): - usr = getattr(spider, 'http_user', '') - pwd = getattr(spider, 'http_pass', '') - if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - - def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth +from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..a6cfca656 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_auth.py similarity index 83% rename from tests/test_downloadermiddleware_httpauth.py rename to tests/test_downloadermiddleware_auth.py index 425a5cc79..78fd120be 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.downloadermiddlewares.auth import AuthMiddleware from scrapy.spiders import Spider @@ -10,10 +10,10 @@ class TestSpider(Spider): http_pass = 'bar' -class HttpAuthMiddlewareTest(unittest.TestCase): +class AuthMiddlewareTest(unittest.TestCase): def setUp(self): - self.mw = HttpAuthMiddleware() + self.mw = AuthMiddleware() self.spider = TestSpider('foo') self.mw.spider_opened(self.spider) From 213da6362103507f3587564939761a370761ff32 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Tue, 8 Sep 2015 00:02:49 -0400 Subject: [PATCH 02/29] fixed url generation --- scrapy/downloadermiddlewares/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py index b832d569a..211d2ecfc 100644 --- a/scrapy/downloadermiddlewares/auth.py +++ b/scrapy/downloadermiddlewares/auth.py @@ -42,5 +42,5 @@ class AuthMiddleware(object): request.headers['Authorization'] = basic_auth_header(url.username, url.password) # no credentials in new url - new_url = url.scheme + '://' + url.hostname + url.path + new_url = url.scheme + '://' + request.url.split('@')[-1] return request.replace(url=new_url) From acf08cb7a2b9cfaeac5e32e30e217d6b08d2b459 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Tue, 8 Sep 2015 00:12:49 -0400 Subject: [PATCH 03/29] replaced httpauth with auth in contrib --- scrapy/contrib/downloadermiddleware/httpauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py index a37ffa0dc..493a4cf6f 100644 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -4,4 +4,4 @@ warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecat "use `scrapy.downloadermiddlewares.httpauth` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.downloadermiddlewares.httpauth import * +from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware From f80b8514b9860b63e421f04a3f84bc7d79e996c8 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Mon, 16 May 2016 13:44:42 -0400 Subject: [PATCH 04/29] fixed deprecation warnings --- scrapy/contrib/downloadermiddleware/httpauth.py | 7 +++++-- scrapy/downloadermiddlewares/httpauth.py | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py index 493a4cf6f..19fc303b5 100644 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -1,7 +1,10 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.httpauth` instead", + "use `scrapy.downloadermiddlewares.auth` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.downloadermiddlewares.auth import AuthMiddleware + +HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 78adc8706..12b4372e5 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -4,4 +4,7 @@ warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, " "use `scrapy.downloadermiddlewares.auth` instead", ScrapyDeprecationWarning) -from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.downloadermiddlewares.auth import AuthMiddleware + +HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) From 05952d00d96151672872ea57c27c78ba8e149bd2 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Mon, 16 May 2016 13:44:53 -0400 Subject: [PATCH 05/29] fixed settings --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a6cfca656..18d5ebbbb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, From 2c8441957e55d96e1667d06d7e7ab941bf831e35 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Sat, 21 May 2016 04:46:02 -0400 Subject: [PATCH 06/29] added unit tests --- tests/test_downloadermiddleware_auth.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index 78fd120be..9b2e0e5bd 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -30,3 +30,18 @@ class AuthMiddlewareTest(unittest.TestCase): headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEquals(req.headers['Authorization'], b'Digest 123') + + def test_auth_from_http_url(self): + req = Request('http://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertTrue('@' not in new_req.url) + + def test_auth_from_ftp_url(self): + req = Request('ftp://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertTrue('ftp_user' in new_req.meta) + self.assertTrue('ftp_password' in new_req.meta) + self.assertTrue('@' not in new_req.url) From 0d9e654f431e7af22eeb8f1c2ba5f79c743d4a21 Mon Sep 17 00:00:00 2001 From: Umair Ashraf Date: Fri, 3 Jun 2016 18:06:22 -0400 Subject: [PATCH 07/29] fixed tests refs 1466#discussion_r64992503 --- tests/test_downloadermiddleware_auth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index 9b2e0e5bd..d5b5b4020 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -36,12 +36,12 @@ class AuthMiddlewareTest(unittest.TestCase): new_req = self.mw.process_request(req, self.spider) assert new_req is not None self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertTrue('@' not in new_req.url) + self.assertEquals(new_req.url, 'http://scrapytest.org/') def test_auth_from_ftp_url(self): req = Request('ftp://username:password@scrapytest.org/') new_req = self.mw.process_request(req, self.spider) assert new_req is not None - self.assertTrue('ftp_user' in new_req.meta) - self.assertTrue('ftp_password' in new_req.meta) - self.assertTrue('@' not in new_req.url) + self.assertIn('ftp_user', new_req.meta) + self.assertIn('ftp_password', new_req.meta) + self.assertNotIn('@', new_req.url) From fb57c71ab49e4797e276791711f714171fc64bd4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 18 Oct 2016 12:45:02 +0200 Subject: [PATCH 08/29] Refactor http/ftp URL tests, use urlpase_cached + add more tests --- scrapy/downloadermiddlewares/auth.py | 69 +++++++++---- tests/test_downloadermiddleware_auth.py | 129 +++++++++++++++++++++++- 2 files changed, 177 insertions(+), 21 deletions(-) diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py index 211d2ecfc..b1242c69d 100644 --- a/scrapy/downloadermiddlewares/auth.py +++ b/scrapy/downloadermiddlewares/auth.py @@ -1,19 +1,38 @@ """ -HTTP basic auth downloader middleware +HTTP/FTP Authorization downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from six.moves.urllib.parse import urlunparse from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.utils.httpobj import urlparse_cached -from six.moves.urllib.parse import urlparse + +def credstrip_url(parsed_url): + """Strip username and password from an urlparse'd URL""" + return urlunparse(( + parsed_url.scheme, + parsed_url.netloc.split('@')[-1], + parsed_url.path, + parsed_url.params, + parsed_url.query, + parsed_url.fragment)) class AuthMiddleware(object): - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" + """ + Populate authorization credentials for HTTP and FTP requests. + + For http(s):// requests, set Basic HTTP Authorization header, + either from http_user and http_pass spider attributes, + or from URL netloc parsing. + + Also handle FTP credentials from ftp://user:password@... URLs, + populating request's meta accordingly. + """ @classmethod def from_crawler(cls, crawler): @@ -28,19 +47,33 @@ class AuthMiddleware(object): self.auth = basic_auth_header(usr, pwd) def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and 'Authorization' not in request.headers: - request.headers['Authorization'] = auth + url = urlparse_cached(request) + if url.scheme.startswith('http'): + # do not override Auth header set priorly + if 'Authorization' in request.headers: + return - # credentials from url are supposed to override spider settings - url = urlparse(request.url) - if url.username and url.password: - if url.scheme.startswith('ftp'): - request.meta['ftp_user'] = url.username - request.meta['ftp_password'] = url.password - elif url.scheme.startswith('http'): - request.headers['Authorization'] = basic_auth_header(url.username, url.password) + new_url = None + # credentials from URL override spider attributes + if url.username or url.password: + auth = basic_auth_header(url.username, url.password) - # no credentials in new url - new_url = url.scheme + '://' + request.url.split('@')[-1] - return request.replace(url=new_url) + # no credentials in new url + new_url = credstrip_url(url) + + else: + auth = getattr(self, 'auth', None) + + if auth: + request.headers['Authorization'] = auth + if new_url: + return request.replace(url=new_url) + + elif url.scheme.startswith('ftp'): + if url.username or url.password: + # priorly set credentials take precedence + request.meta.setdefault('ftp_user', url.username) + request.meta.setdefault('ftp_password', url.password) + + # no credentials in new url + return request.replace(url=credstrip_url(url)) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index d5b5b4020..c58eabcb6 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -10,6 +10,32 @@ class TestSpider(Spider): http_pass = 'bar' +class NoAuthTestSpider(Spider): + """A test spider that does not set http auth atttributes""" + + +class AuthMiddlewareNoAuthTest(unittest.TestCase): + + def setUp(self): + self.mw = AuthMiddleware() + self.spider = NoAuthTestSpider('bar') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_no_auth_http(self): + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_no_auth_ftp(self): + req = Request('ftp://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn('ftp_user', req.meta) + self.assertNotIn('ftp_password', req.meta) + + class AuthMiddlewareTest(unittest.TestCase): def setUp(self): @@ -20,6 +46,9 @@ class AuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw + +class AuthMiddlewareHttpAuthTest(AuthMiddlewareTest): + def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None @@ -38,10 +67,104 @@ class AuthMiddlewareTest(unittest.TestCase): self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') self.assertEquals(new_req.url, 'http://scrapytest.org/') + def test_auth_from_https_url(self): + req = Request('https://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'https://scrapytest.org/') + + def test_auth_from_http_url_no_spider_attrs(self): + class AnotherTestSpider(Spider): + pass + req = Request('http://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, AnotherTestSpider('bar')) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_https_url_no_spider_attrs(self): + class AnotherTestSpider(Spider): + pass + req = Request('https://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, AnotherTestSpider('bar')) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'https://scrapytest.org/') + + def test_auth_from_http_url_empty_pass(self): + req = Request('http://username:@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_http_url_pass_none(self): + req = Request('http://username@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6Tm9uZQ==') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_http_url_empty_user(self): + req = Request('http://:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic OnBhc3N3b3Jk') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + +class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): + + def test_no_auth_from_ftp_url_meta_unchanged(self): + usr, pwd = 'u', 'p' + req = Request('ftp://scrapytest.org/', + meta={"ftp_user": usr, "ftp_password": pwd}) + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.meta['ftp_user'], usr) + self.assertEquals(req.meta['ftp_password'], pwd) + + def test_auth_from_ftp_url_meta_unchanged(self): + """Request's meta credentials are kept as-is, + but URL is stripped from credentials + """ + usr, pwd = 'u', 'p' + req = Request('ftp://username:password@scrapytest.org/', + meta={"ftp_user": usr, "ftp_password": pwd}) + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.meta['ftp_user'], usr) + self.assertEquals(new_req.meta['ftp_password'], pwd) + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url(self): req = Request('ftp://username:password@scrapytest.org/') new_req = self.mw.process_request(req, self.spider) assert new_req is not None - self.assertIn('ftp_user', new_req.meta) - self.assertIn('ftp_password', new_req.meta) - self.assertNotIn('@', new_req.url) + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_empty_user(self): + req = Request('ftp://:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], '') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_empty_pass(self): + req = Request('ftp://username:@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], '') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_pass_none(self): + req = Request('ftp://username@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], None) + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') From 41b03a567c2e0bd415e2d2ed0d72a10c53014a7f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 18 Oct 2016 15:38:06 +0200 Subject: [PATCH 09/29] Update default settings to point to new AuthMiddleware --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..a6cfca656 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, From aa36c8bc3a4a7fc0a228ef153ddc0a25fe7b506d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 18 Oct 2016 15:38:32 +0200 Subject: [PATCH 10/29] Update docs on new AuthMiddleware --- docs/faq.rst | 2 +- docs/topics/downloader-middleware.rst | 36 ++++++++++++++++++++------- docs/topics/settings.rst | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 415331515..d3efc09fc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ See previous question. Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- -Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`. +Yes, see :class:`~scrapy.downloadermiddlewares.auth.AuthMiddleware`. Why does Scrapy download pages in English instead of my native language? ------------------------------------------------------------------------ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15069e56e..01927e584 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -278,21 +278,30 @@ DownloadTimeoutMiddleware :reqmeta:`download_timeout` Request.meta key; this is supported even when DownloadTimeoutMiddleware is disabled. -HttpAuthMiddleware +AuthMiddleware ------------------ -.. module:: scrapy.downloadermiddlewares.httpauth - :synopsis: HTTP Auth downloader middleware +.. module:: scrapy.downloadermiddlewares.auth + :synopsis: HTTP/FTP Auth downloader middleware -.. class:: HttpAuthMiddleware +.. class:: AuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware populates authentication credentials for HTTP and FTP requests. - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication (`Basic access authentication`_, aka. HTTP auth), + you have two options: - Example:: + - either set the ``http_user`` and ``http_pass`` attributes of the spider(s) + for which you need HTTP auth, + and these will be applied to all http(s):// requests, + - or, on a per-Request basis, include credentials using the standard URL + syntax of ``http://username:password@www.example.com/index.html`` + + To populate credentials for FTP requests, use URLs in the form of + ``ftp://user:password@www.example.com/document.txt``, which will + set the ``meta`` dict ``ftp_user`` and ``ftp_password`` values. + + Example of enabling HTTP Basic Auth for all HTTP requests:: from scrapy.spiders import CrawlSpider @@ -306,6 +315,15 @@ HttpAuthMiddleware .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication +HttpAuthMiddleware +------------------ + +.. module:: scrapy.downloadermiddlewares.httpauth + :synopsis: HTTP Auth downloader middleware + +.. class:: HttpAuthMiddleware + + This middleware is deprecated and redirects to :class:`~.AuthMiddleware`. HttpCacheMiddleware ------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..f6493d155 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -457,7 +457,7 @@ Default:: { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, From 588705f3e8c266cb35c10d6e53ec3583b6322f9e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Oct 2016 13:13:00 +0200 Subject: [PATCH 11/29] Support percent-encoded delimiters in userinfo --- scrapy/downloadermiddlewares/auth.py | 14 ++++++++++---- tests/test_downloadermiddleware_auth.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py index b1242c69d..64239ab45 100644 --- a/scrapy/downloadermiddlewares/auth.py +++ b/scrapy/downloadermiddlewares/auth.py @@ -3,7 +3,7 @@ HTTP/FTP Authorization downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -from six.moves.urllib.parse import urlunparse +from six.moves.urllib.parse import unquote, urlunparse from w3lib.http import basic_auth_header @@ -22,6 +22,11 @@ def credstrip_url(parsed_url): parsed_url.fragment)) +def _unquote(s): + if s is not None: + return unquote(s) + + class AuthMiddleware(object): """ Populate authorization credentials for HTTP and FTP requests. @@ -56,7 +61,8 @@ class AuthMiddleware(object): new_url = None # credentials from URL override spider attributes if url.username or url.password: - auth = basic_auth_header(url.username, url.password) + auth = basic_auth_header(_unquote(url.username), + _unquote(url.password)) # no credentials in new url new_url = credstrip_url(url) @@ -72,8 +78,8 @@ class AuthMiddleware(object): elif url.scheme.startswith('ftp'): if url.username or url.password: # priorly set credentials take precedence - request.meta.setdefault('ftp_user', url.username) - request.meta.setdefault('ftp_password', url.password) + request.meta.setdefault('ftp_user', _unquote(url.username)) + request.meta.setdefault('ftp_password', _unquote(url.password)) # no credentials in new url return request.replace(url=credstrip_url(url)) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index c58eabcb6..6f7a583ef 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -145,6 +145,22 @@ class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): self.assertEquals(req.meta['ftp_password'], 'password') self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url_encoded_delims_user(self): + req = Request('ftp://username%3A:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username:') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_encoded_delims_password(self): + req = Request('ftp://username:pass%40word@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], 'pass@word') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url_empty_user(self): req = Request('ftp://:password@scrapytest.org/') new_req = self.mw.process_request(req, self.spider) From 6b01f77bb11bfc895751f00a2258412e2de4037e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 10:16:46 +0100 Subject: [PATCH 12/29] Revert change to default settings for Auth middleware --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a6cfca656..18d5ebbbb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, From 9fc45cccb538366d1da123dbfdc2a29e3e7d7631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 6 Apr 2020 02:17:11 +0200 Subject: [PATCH 13/29] Implement UriUserinfoMiddleware --- docs/faq.rst | 2 +- docs/topics/downloader-middleware.rst | 543 +++++++++++++----- docs/topics/settings.rst | 493 ++++++++++++++-- .../contrib/downloadermiddleware/httpauth.py | 7 +- scrapy/downloadermiddlewares/auth.py | 85 --- scrapy/downloadermiddlewares/httpauth.py | 37 +- scrapy/downloadermiddlewares/uriuserinfo.py | 49 ++ scrapy/settings/default_settings.py | 1 + tests/test_downloadermiddleware_auth.py | 186 ------ tests/test_downloadermiddleware_httpauth.py | 32 ++ .../test_downloadermiddleware_uriuserinfo.py | 72 +++ 11 files changed, 1006 insertions(+), 501 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/auth.py create mode 100644 scrapy/downloadermiddlewares/uriuserinfo.py delete mode 100644 tests/test_downloadermiddleware_auth.py create mode 100644 tests/test_downloadermiddleware_httpauth.py create mode 100644 tests/test_downloadermiddleware_uriuserinfo.py diff --git a/docs/faq.rst b/docs/faq.rst index d3efc09fc..415331515 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ See previous question. Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- -Yes, see :class:`~scrapy.downloadermiddlewares.auth.AuthMiddleware`. +Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`. Why does Scrapy download pages in English instead of my native language? ------------------------------------------------------------------------ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 01927e584..b9b81aa54 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it -in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { @@ -52,11 +52,17 @@ as its value. For example, if you want to disable the user-agent middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _topics-downloader-middleware-custom: + Writing your own downloader middleware ====================================== -Each middleware component is a Python class that defines one or -more of the following methods: +Each downloader middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.downloadermiddlewares @@ -157,6 +163,17 @@ more of the following methods: :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -182,7 +199,7 @@ CookiesMiddleware This middleware enables working with sites that require cookies, such as those that use sessions. It keeps track of cookies sent by web servers, and - send them back on subsequent requests (from that spider), just like web + sends them back on subsequent requests (from that spider), just like web browsers do. The following settings can be used to configure the cookie middleware: @@ -226,6 +243,15 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. + +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. + .. setting:: COOKIES_DEBUG COOKIES_DEBUG @@ -233,19 +259,19 @@ COOKIES_DEBUG Default: ``False`` -If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie`` -header) and all cookies received in responses (ie. ``Set-Cookie`` header). +If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie`` +header) and all cookies received in responses (i.e. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - 2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ Set-Cookie: ip_isocode=US Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) (referer: None) + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [...] @@ -278,43 +304,6 @@ DownloadTimeoutMiddleware :reqmeta:`download_timeout` Request.meta key; this is supported even when DownloadTimeoutMiddleware is disabled. -AuthMiddleware ------------------- - -.. module:: scrapy.downloadermiddlewares.auth - :synopsis: HTTP/FTP Auth downloader middleware - -.. class:: AuthMiddleware - - This middleware populates authentication credentials for HTTP and FTP requests. - - To enable HTTP authentication (`Basic access authentication`_, aka. HTTP auth), - you have two options: - - - either set the ``http_user`` and ``http_pass`` attributes of the spider(s) - for which you need HTTP auth, - and these will be applied to all http(s):// requests, - - or, on a per-Request basis, include credentials using the standard URL - syntax of ``http://username:password@www.example.com/index.html`` - - To populate credentials for FTP requests, use URLs in the form of - ``ftp://user:password@www.example.com/document.txt``, which will - set the ``meta`` dict ``ftp_user`` and ``ftp_password`` values. - - Example of enabling HTTP Basic Auth for all HTTP requests:: - - from scrapy.spiders import CrawlSpider - - class SomeIntranetSiteSpider(CrawlSpider): - - http_user = 'someuser' - http_pass = 'somepass' - name = 'intranet.example.com' - - # .. rest of the spider code omitted ... - -.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication - HttpAuthMiddleware ------------------ @@ -323,7 +312,30 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware is deprecated and redirects to :class:`~.AuthMiddleware`. + This middleware authenticates all requests generated from certain spiders + using `Basic access authentication`_ (aka. HTTP auth). + + To enable HTTP authentication from certain spiders, set the ``http_user`` + and ``http_pass`` attributes of those spiders. + + Example:: + + from scrapy.spiders import CrawlSpider + + class SomeIntranetSiteSpider(CrawlSpider): + name = 'intranet.example.com' + http_user = 'someuser' + http_pass = 'somepass' + + .. reqmeta:: http_user + .. reqmeta:: http_pass + + You can alternatively specify ``http_user`` and ``http_pass`` in + :attr:`Request.meta `, or use + :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`. + +.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication + HttpCacheMiddleware ------------------- @@ -336,13 +348,13 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with two HTTP cache storage backends: + Scrapy ships with three HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` - setting. Or you can also implement your own storage backend. + setting. Or you can also :ref:`implement your own storage backend. ` Scrapy ships with two HTTP cache policies: @@ -354,26 +366,27 @@ HttpCacheMiddleware .. reqmeta:: dont_cache - You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`. + You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``. + +.. module:: scrapy.extensions.httpcache + :noindex: .. _httpcache-policy-dummy: Dummy policy (default) ~~~~~~~~~~~~~~~~~~~~~~ -This policy has no awareness of any HTTP Cache-Control directives. -Every request and its corresponding response are cached. When the same -request is seen again, the response is returned without transferring -anything from the Internet. +.. class:: DummyPolicy -The Dummy policy is useful for testing spiders faster (without having -to wait for downloads every time) and for trying your spider offline, -when an Internet connection is not available. The goal is to be able to -"replay" a spider run *exactly as it ran before*. + This policy has no awareness of any HTTP Cache-Control directives. + Every request and its corresponding response are cached. When the same + request is seen again, the response is returned without transferring + anything from the Internet. -In order to use this policy, set: - -* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy`` + The Dummy policy is useful for testing spiders faster (without having + to wait for downloads every time) and for trying your spider offline, + when an Internet connection is not available. The goal is to be able to + "replay" a spider run *exactly as it ran before*. .. _httpcache-policy-rfc2616: @@ -381,45 +394,44 @@ In order to use this policy, set: RFC2616 policy ~~~~~~~~~~~~~~ -This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP -Cache-Control awareness, aimed at production and used in continuous -runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). +.. class:: RFC2616Policy -what is implemented: + This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP + Cache-Control awareness, aimed at production and used in continuous + runs to avoid downloading unmodified data (to save bandwidth and speed up + crawls). -* Do not attempt to store responses/requests with `no-store` cache-control directive set -* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses -* Compute freshness lifetime from `max-age` cache-control directive -* Compute freshness lifetime from `Expires` response header -* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox) -* Compute current age from `Age` response header -* Compute current age from `Date` header -* Revalidate stale responses based on `Last-Modified` response header -* Revalidate stale responses based on `ETag` response header -* Set `Date` header for any received response missing it -* Support `max-stale` cache-control directive in requests + What is implemented: - This allows spiders to be configured with the full RFC2616 cache policy, - but avoid revalidation on a request-by-request basis, while remaining - conformant with the HTTP spec. + * Do not attempt to store responses/requests with ``no-store`` cache-control directive set + * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses + * Compute freshness lifetime from ``max-age`` cache-control directive + * Compute freshness lifetime from ``Expires`` response header + * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox) + * Compute current age from ``Age`` response header + * Compute current age from ``Date`` header + * Revalidate stale responses based on ``Last-Modified`` response header + * Revalidate stale responses based on ``ETag`` response header + * Set ``Date`` header for any received response missing it + * Support ``max-stale`` cache-control directive in requests - Example: + This allows spiders to be configured with the full RFC2616 cache policy, + but avoid revalidation on a request-by-request basis, while remaining + conformant with the HTTP spec. - Add `Cache-Control: max-stale=600` to Request headers to accept responses that - have exceeded their expiration time by no more than 600 seconds. + Example: - See also: RFC2616, 14.9.3 + Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that + have exceeded their expiration time by no more than 600 seconds. -what is missing: + See also: RFC2616, 14.9.3 -* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 -* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 -* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 -* ... probably others .. + What is missing: -In order to use this policy, set: - -* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy`` + * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 + * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 + * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 + * ... probably others .. .. _httpcache-storage-fs: @@ -427,67 +439,102 @@ In order to use this policy, set: Filesystem storage backend (default) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -File system storage backend is available for the HTTP cache middleware. +.. class:: FilesystemCacheStorage -In order to use this storage backend, set: + File system storage backend is available for the HTTP cache middleware. -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage`` + Each request/response pair is stored in a different directory containing + the following files: -Each request/response pair is stored in a different directory containing -the following files: + * ``request_body`` - the plain request body - * ``request_body`` - the plain request body - * ``request_headers`` - the request headers (in raw HTTP format) - * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) - * ``meta`` - some metadata of this cache resource in Python ``repr()`` format - (grep-friendly format) - * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more - efficient deserialization + * ``request_headers`` - the request headers (in raw HTTP format) -The directory name is made from the request fingerprint (see -``scrapy.utils.request.fingerprint``), and one level of subdirectories is -used to avoid creating too many files into the same directory (which is -inefficient in many file systems). An example directory could be:: + * ``response_body`` - the plain response body - /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 + * ``response_headers`` - the request headers (in raw HTTP format) + + * ``meta`` - some metadata of this cache resource in Python ``repr()`` + format (grep-friendly format) + + * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more + efficient deserialization + + The directory name is made from the request fingerprint (see + ``scrapy.utils.request.fingerprint``), and one level of subdirectories is + used to avoid creating too many files into the same directory (which is + inefficient in many file systems). An example directory could be:: + + /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 .. _httpcache-storage-dbm: DBM storage backend ~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.13 +.. class:: DbmCacheStorage -A DBM_ storage backend is also available for the HTTP cache middleware. + .. versionadded:: 0.13 -By default, it uses the anydbm_ module, but you can change it with the -:setting:`HTTPCACHE_DBM_MODULE` setting. + A DBM_ storage backend is also available for the HTTP cache middleware. -In order to use this storage backend, set: + By default, it uses the :mod:`dbm`, but you can change it with the + :setting:`HTTPCACHE_DBM_MODULE` setting. -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage`` +.. _httpcache-storage-custom: -.. _httpcache-storage-leveldb: +Writing your own storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -LevelDB storage backend -~~~~~~~~~~~~~~~~~~~~~~~ +You can implement a cache storage backend by creating a Python class that +defines the methods described below. -.. versionadded:: 0.23 +.. module:: scrapy.extensions.httpcache -A LevelDB_ storage backend is also available for the HTTP cache middleware. +.. class:: CacheStorage -This backend is not recommended for development because only one process can -access LevelDB databases at the same time, so you can't run a crawl and open -the scrapy shell in parallel for the same spider. + .. method:: open_spider(spider) -In order to use this storage backend: + This method gets called after a spider has been opened for crawling. It handles + the :signal:`open_spider ` signal. -* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage`` -* install `LevelDB python bindings`_ like ``pip install leveldb`` + :param spider: the spider which has been opened + :type spider: :class:`~scrapy.spiders.Spider` object -.. _LevelDB: https://github.com/google/leveldb -.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb + .. method:: close_spider(spider) + + This method gets called after a spider has been closed. It handles + the :signal:`close_spider ` signal. + + :param spider: the spider which has been closed + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: retrieve_response(spider, request) + + Return response if present in cache, or ``None`` otherwise. + + :param spider: the spider which generated the request + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the request to find cached response for + :type request: :class:`~scrapy.http.Request` object + + .. method:: store_response(spider, request, response) + + Store the given response in the cache. + + :param spider: the spider for which the response is intended + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the corresponding request the spider generated + :type request: :class:`~scrapy.http.Request` object + + :param response: the response to store in the cache + :type response: :class:`~scrapy.http.Response` object + +In order to use your storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class. HTTPCache middleware settings @@ -583,7 +630,7 @@ HTTPCACHE_DBM_MODULE .. versionadded:: 0.13 -Default: ``'anydbm'`` +Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. @@ -623,13 +670,13 @@ Default: ``False`` If enabled, will cache pages unconditionally. A spider may wish to have all responses available in the cache, for -future use with `Cache-Control: max-stale`, for instance. The +future use with ``Cache-Control: max-stale``, for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. -This setting still respects `Cache-Control: no-store` directives in responses. -If you don't want that, filter `no-store` out of the Cache-Control headers in -responses you feedto the cache middleware. +This setting still respects ``Cache-Control: no-store`` directives in responses. +If you don't want that, filter ``no-store`` out of the Cache-Control headers in +responses you feed to the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -643,7 +690,7 @@ Default: ``[]`` List of Cache-Control directives in responses to be ignored. Sites often set "no-store", "no-cache", "must-revalidate", etc., but get -upset at the traffic a spider can generate if it respects those +upset at the traffic a spider can generate if it actually respects those directives. This allows to selectively ignore Cache-Control directives that are known to be unimportant for the sites being crawled. @@ -662,6 +709,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. + This middleware also supports decoding `brotli-compressed`_ responses, + provided `brotlipy`_ is installed. + +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotlipy: https://pypi.org/project/brotlipy/ + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -675,16 +728,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_ - HttpProxyMiddleware ------------------- @@ -708,7 +751,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -728,6 +773,17 @@ RedirectMiddleware The urls which the request goes through (while being redirected) can be found in the ``redirect_urls`` :attr:`Request.meta ` key. +.. reqmeta:: redirect_reasons + +The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the +``redirect_reasons`` :attr:`Request.meta ` key. For +example: ``[301, 302, 307, 'meta refresh']``. + +The format of a reason depends on the middleware that handled the corresponding +redirect. For example, :class:`RedirectMiddleware` indicates the triggering +response status code as an integer, while :class:`MetaRefreshMiddleware` +always uses the ``'meta refresh'`` string as reason. + The :class:`RedirectMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -776,7 +832,7 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be follow for a single request. +The maximum number of redirections that will be followed for a single request. MetaRefreshMiddleware --------------------- @@ -789,10 +845,12 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` -This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect` -and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware` +This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, +:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described +for :class:`RedirectMiddleware` MetaRefreshMiddleware settings @@ -809,6 +867,19 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``[]`` + +Meta tags within these tags are ignored. + +.. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY @@ -833,8 +904,6 @@ RetryMiddleware Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. -Once there are no more failed pages to retry, this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -871,12 +940,17 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. +Maximum number of retries can also be specified per-request using +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +When initialized, the :reqmeta:`max_retry_times` meta key takes higher +precedence over the :setting:`RETRY_TIMES` setting. + .. setting:: RETRY_HTTP_CODES RETRY_HTTP_CODES ^^^^^^^^^^^^^^^^ -Default: ``[500, 502, 503, 504, 408]`` +Default: ``[500, 502, 503, 504, 522, 524, 408, 429]`` Which HTTP response codes to retry. Other errors (DNS lookup issues, connections lost, etc) are always retried. @@ -902,6 +976,24 @@ RobotsTxtMiddleware To make sure Scrapy respects robots.txt make sure the middleware is enabled and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the + user agent string to use for matching in the robots.txt_ file. If it + is ``None``, the User-Agent header you are sending with the request or the + :setting:`USER_AGENT` setting (in that order) will be used for determining + the user agent to use in the robots.txt_ file. + + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` + * :ref:`Reppy ` + * :ref:`Robotexclusionrulesparser ` + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser `. + .. reqmeta:: dont_obey_robotstxt If :attr:`Request.meta ` has @@ -909,6 +1001,129 @@ If :attr:`Request.meta ` has the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. +Parsers vary in several aspects: + +* Language of implementation + +* Supported specification + +* Support for wildcard matching + +* Usage of `length based rule `_: + in particular for ``Allow`` and ``Disallow`` directives, where the most + specific rule based on the length of the path trumps the less specific + (shorter) rule + +Performance comparison of different parsers is available at `the following link +`_. + +.. _protego-parser: + +Protego parser +~~~~~~~~~~~~~~ + +Based on `Protego `_: + +* implemented in Python + +* is compliant with `Google's Robots.txt Specification + `_ + +* supports wildcard matching + +* uses the length based rule + +Scrapy uses this parser by default. + +.. _python-robotfileparser: + +RobotFileParser +~~~~~~~~~~~~~~~ + +Based on `RobotFileParser +`_: + +* is Python's built-in robots.txt_ parser + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* lacks support for wildcard matching + +* doesn't use the length based rule + +It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. + +In order to use this parser, set: + +* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` + +.. _reppy-parser: + +Reppy parser +~~~~~~~~~~~~ + +Based on `Reppy `_: + +* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ + `_ + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* supports wildcard matching + +* uses the length based rule + +Native implementation, provides better speed than Protego. + +In order to use this parser: + +* Install `Reppy `_ by running ``pip install reppy`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.ReppyRobotParser`` + +.. _rerp-parser: + +Robotexclusionrulesparser +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Based on `Robotexclusionrulesparser `_: + +* implemented in Python + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* supports wildcard matching + +* doesn't use the length based rule + +In order to use this parser: + +* Install `Robotexclusionrulesparser `_ by running + ``pip install robotexclusionrulesparser`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.RerpRobotParser`` + +.. _support-for-new-robots-parser: + +Implementing support for a new parser +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can implement support for a new robots.txt_ parser by subclassing +the abstract base class :class:`~scrapy.robotstxt.RobotParser` and +implementing the methods described below. + +.. module:: scrapy.robotstxt + :synopsis: robots.txt parser interface and implementations + +.. autoclass:: RobotParser + :members: + +.. _robots.txt: https://www.robotstxt.org/ DownloaderStats --------------- @@ -924,6 +1139,16 @@ DownloaderStats To use this middleware you must enable the :setting:`DOWNLOADER_STATS` setting. + +UriUserinfoMiddleware +--------------------- + +.. module:: scrapy.downloadermiddlewares.uriuserinfo + :synopsis: URI Userinfo Middleware + +.. autoclass:: UriUserinfoMiddleware + + UserAgentMiddleware ------------------- @@ -934,7 +1159,7 @@ UserAgentMiddleware Middleware that allows spiders to override the default user agent. - In order for a spider to override the default user agent, its `user_agent` + In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. .. _ajaxcrawl-middleware: @@ -948,7 +1173,7 @@ AjaxCrawlMiddleware Middleware that finds 'AJAX crawlable' page variants based on meta-fragment html tag. See - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started + https://developers.google.com/search/docs/ajax-crawling/docs/getting-started for more info. .. note:: @@ -976,8 +1201,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ @@ -987,5 +1220,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f6493d155..7d3026455 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -30,6 +30,8 @@ Python `import search path`_. .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path +.. _populating-settings: + Populating the settings ======================= @@ -122,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in extensions, middlewares and item pipelines:: - class MyExtension(object): + class MyExtension: def __init__(self, log_is_enabled=False): if log_is_enabled: print("log is enabled!") @@ -178,6 +180,44 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_ENDPOINT_URL + +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like storage, for example Minio or s3.scality. + +.. setting:: AWS_USE_SSL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with +S3 or S3-like storage. By default SSL will be used. + +.. setting:: AWS_VERIFY + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default +SSL verification will occur. + +.. setting:: AWS_REGION_NAME + +AWS_REGION_NAME +--------------- + +Default: ``None`` + +The name of the region associated with the AWS client. + .. setting:: BOT_NAME BOT_NAME @@ -186,8 +226,7 @@ BOT_NAME Default: ``'scrapybot'`` The name of the bot implemented by this Scrapy project (also known as the -project name). This will be used to construct the User-Agent by default, and -also for logging. +project name). This name will be used for the logging too. It's automatically populated with your project name when you create your project with the :command:`startproject` command. @@ -209,7 +248,7 @@ CONCURRENT_REQUESTS Default: ``16`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed by the Scrapy downloader. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -219,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN Default: ``8`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. See also: :ref:`topics-autothrottle` and its @@ -233,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP Default: ``0`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single IP. If non-zero, the :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is used instead. In other words, concurrency limits will be applied per IP, not @@ -290,16 +329,16 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the request priority based on its depth: +An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of +a :class:`~scrapy.http.Request` based on its depth. -- if zero (default), no priority adjustment is made from depth -- **a positive value will decrease the priority, i.e. higher depth - requests will be processed later** ; this is commonly used when doing - breadth-first crawls (BFO) -- a negative value will increase priority, i.e., higher depth requests - will be processed sooner (DFO) +The priority of a request is adjusted as follows:: -See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. + request.priority = request.priority - ( depth * DEPTH_PRIORITY ) + +As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request +priority (BFO), while negative values increase request priority (DFO). See +also :ref:`faq-bfo-dfo`. .. note:: @@ -307,17 +346,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` and :setting:`RETRY_PRIORITY_ADJUST`. -.. setting:: DEPTH_STATS - -DEPTH_STATS ------------ - -Default: ``True`` - -Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` - -Whether to collect maximum depth stats. - .. setting:: DEPTH_STATS_VERBOSE DEPTH_STATS_VERBOSE @@ -348,6 +376,21 @@ Default: ``10000`` DNS in-memory cache size. +.. setting:: DNS_RESOLVER + +DNS_RESOLVER +------------ + +.. versionadded:: 2.0 + +Default: ``'scrapy.resolver.CachingThreadedResolver'`` + +The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver`` +supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting, +but works only with IPv4 addresses. Scrapy provides an alternative resolver, +``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not +take the :setting:`DNS_TIMEOUT` setting into account. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -408,9 +451,29 @@ or even enable client-side authentication (and various other things). which uses the platform's certificates to validate remote endpoints. **This is only available if you use Twisted>=14.0.** -If you do use a custom ContextFactory, make sure it accepts a ``method`` -parameter at init (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`). +If you do use a custom ContextFactory, make sure its ``__init__`` method +accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping +:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging`` +parameter (``bool``) and a ``tls_ciphers`` parameter (see +:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`). + +.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS + +DOWNLOADER_CLIENT_TLS_CIPHERS +----------------------------- + +Default: ``'DEFAULT'`` + +Use this setting to customize the TLS/SSL ciphers used by the default +HTTP/1.1 downloader. + +The setting should contain a string in the `OpenSSL cipher list format`_, +these ciphers will be used as client ciphers. Changing this setting may be +necessary to access certain HTTPS websites: for example, you may need to use +``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a +specific cipher that is not included in ``DEFAULT`` if a website requires it. + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT .. setting:: DOWNLOADER_CLIENT_TLS_METHOD @@ -438,6 +501,20 @@ This setting must be one of these string values: We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 or above (Twisted>=14.0 if you can). +.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING + +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING +------------------------------------- + +Default: ``False`` + +Setting this to ``True`` will enable DEBUG level messages about TLS connection +parameters after establishing HTTPS connections. The kind of information logged +depends on the versions of OpenSSL and pyOpenSSL. + +This setting is only used for the default +:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES @@ -457,7 +534,8 @@ Default:: { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, @@ -509,6 +587,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced per ip address instead of per domain. +.. _spider-download_delay-attribute: + You can also change this setting per spider by setting ``download_delay`` spider attribute. @@ -570,7 +650,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024MB) +Default: ``1073741824`` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -591,7 +671,7 @@ If you want to disable it set to 0. DOWNLOAD_WARNSIZE ----------------- -Default: `33554432` (32MB) +Default: ``33554432`` (32MB) The response size (in bytes) that downloader will start to warn. @@ -605,6 +685,32 @@ If you want to disable it set to 0. This feature needs Twisted >= 11.1. +.. setting:: DOWNLOAD_FAIL_ON_DATALOSS + +DOWNLOAD_FAIL_ON_DATALOSS +------------------------- + +Default: ``True`` + +Whether or not to fail on broken responses, that is, declared +``Content-Length`` does not match content sent by the server or chunked +response was not properly finish. If ``True``, these responses raise a +``ResponseFailed([_DataLoss])`` error. If ``False``, these responses +are passed through and the flag ``dataloss`` is added to the response, i.e.: +``'dataloss' in response.flags`` is ``True``. + +Optionally, this can be set per-request basis by using the +:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. + +.. note:: + + A broken response, or data loss error, may happen under several + circumstances, from server misconfiguration to network errors to data + corruption. It is up to the user to decide if it makes sense to process + broken responses considering they may contain partial or incomplete content. + If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -621,6 +727,13 @@ override its ``request_fingerprint`` method. This method should accept scrapy :class:`~scrapy.http.Request` object and return its fingerprint (a string). +You can disable filtering of duplicate requests by setting +:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. +Be very careful about this however, because you can get into crawling loops. +It's usually a better idea to set the ``dont_filter`` parameter to +``True`` on the specific :class:`~scrapy.http.Request` that should not be +filtered. + .. setting:: DUPEFILTER_DEBUG DUPEFILTER_DEBUG @@ -636,11 +749,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r EDITOR ------ -Default: `depends on the environment` +Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows) -The editor to use for editing spiders with the :command:`edit` command. It -defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults -to ``vi`` (on Unix systems) or the IDLE editor (on Windows). +The editor to use for editing spiders with the :command:`edit` command. +Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit` +command will prefer it over the default setting. .. setting:: EXTENSIONS @@ -687,6 +800,57 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FTP_PASSIVE_MODE + +FTP_PASSIVE_MODE +---------------- + +Default: ``True`` + +Whether or not to use passive mode when initiating FTP transfers. + +.. reqmeta:: ftp_password +.. setting:: FTP_PASSWORD + +FTP_PASSWORD +------------ + +Default: ``"guest"`` + +The password to use for FTP connections when there is no ``"ftp_password"`` +in ``Request`` meta. + +It can be overriden in a request in any of the following ways: + +- Specifying ``ftp_password`` in :attr:`Request.meta ` + +- Specifying the password in :attr:`Request.url ` + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) + +.. note:: + Paraphrasing `RFC 1635`_, although it is common to use either the password + "guest" or one's e-mail address for anonymous FTP, + some FTP servers explicitly ask for the user's e-mail address + and will not allow login with the "guest" password. + +.. _RFC 1635: https://tools.ietf.org/html/rfc1635 + +.. reqmeta:: ftp_user +.. setting:: FTP_USER + +FTP_USER +-------- + +Default: ``"anonymous"`` + +The default username to use for FTP connections. + +It can be overriden in a request in any of the following ways: + +- Specifying ``ftp_user`` in :attr:`Request.meta ` + +- Specifying the username in :attr:`Request.url ` + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) .. setting:: ITEM_PIPELINES @@ -750,7 +914,7 @@ LOG_FORMAT Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` -String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available +String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available placeholders. .. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes @@ -768,6 +932,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: :class:`scrapy.logformatter.LogFormatter` + +The class to use for :ref:`formatting log messages ` for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL @@ -786,9 +959,29 @@ LOG_STDOUT Default: ``False`` If ``True``, all standard output (and error) of your process will be redirected -to the log. For example if you ``print 'hello'`` it will appear in the Scrapy +to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. +.. setting:: LOG_SHORT_NAMES + +LOG_SHORT_NAMES +--------------- + +Default: ``False`` + +If ``True``, the logs will just contain the root path. If it is set to ``False`` +then it displays the component responsible for the log output + +.. setting:: LOGSTATS_INTERVAL + +LOGSTATS_INTERVAL +----------------- + +Default: ``60.0`` + +The interval (in seconds) between each logging printout of the stats +by :class:`~scrapy.extensions.logstats.LogStats`. + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED @@ -818,13 +1011,15 @@ Example:: MEMUSAGE_ENABLED ---------------- -Default: ``False`` +Default: ``True`` Scope: ``scrapy.extensions.memusage`` -Whether to enable the memory usage extension that will shutdown the Scrapy -process when it exceeds a memory limit, and also notify by email when that -happened. +Whether to enable the memory usage extension. This extension keeps track of +a peak memory used by the process (it writes it to stats). It can also +optionally shutdown the Scrapy process when it exceeds a memory limit +(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened +(see :setting:`MEMUSAGE_NOTIFY_MAIL`). See :ref:`topics-extensions-ref-memusage`. @@ -879,19 +1074,6 @@ Example:: See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_REPORT - -MEMUSAGE_REPORT ---------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -Whether to send a memory usage report after each spider has been closed. - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -935,7 +1117,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option. If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. -.. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. _wget: https://www.gnu.org/software/wget/manual/wget.html .. setting:: REACTOR_THREADPOOL_MAXSIZE @@ -1006,6 +1188,28 @@ If enabled, Scrapy will respect robots.txt policies. For more information see this option is enabled by default in settings.py file generated by ``scrapy startproject`` command. +.. setting:: ROBOTSTXT_PARSER + +ROBOTSTXT_PARSER +---------------- + +Default: ``'scrapy.robotstxt.ProtegoRobotParser'`` + +The parser backend to use for parsing ``robots.txt`` files. For more information see +:ref:`topics-dlmw-robots`. + +.. setting:: ROBOTSTXT_USER_AGENT + +ROBOTSTXT_USER_AGENT +^^^^^^^^^^^^^^^^^^^^ + +Default: ``None`` + +The user agent string to use for matching in the robots.txt file. If ``None``, +the User-Agent header you are sending with the request or the +:setting:`USER_AGENT` setting (in that order) will be used for determining +the user agent to use in the robots.txt file. + .. setting:: SCHEDULER SCHEDULER @@ -1028,11 +1232,59 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + 1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request: - reason: cannot serialize (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) + +.. setting:: SCHEDULER_DISK_QUEUE + +SCHEDULER_DISK_QUEUE +-------------------- + +Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` + +Type of disk queue that will be used by scheduler. Other available types are +``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +``scrapy.squeues.MarshalLifoDiskQueue``. + +.. setting:: SCHEDULER_MEMORY_QUEUE + +SCHEDULER_MEMORY_QUEUE +---------------------- +Default: ``'scrapy.squeues.LifoMemoryQueue'`` + +Type of in-memory queue used by scheduler. Other available type is: +``scrapy.squeues.FifoMemoryQueue``. + +.. setting:: SCHEDULER_PRIORITY_QUEUE + +SCHEDULER_PRIORITY_QUEUE +------------------------ +Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` + +Type of priority queue used by the scheduler. Another available type is +``scrapy.pqueues.DownloaderAwarePriorityQueue``. +``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than +``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different +domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. + +.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE + +SCRAPER_SLOT_MAX_ACTIVE_SIZE +---------------------------- + +.. versionadded:: 2.0 + +Default: ``5_000_000`` + +Soft limit (in bytes) for response data being processed. + +While the sum of the sizes of all responses being processed is above this value, +Scrapy does not process new requests. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS @@ -1056,7 +1308,7 @@ Default:: 'scrapy.contracts.default.ScrapesContract': 3, } -A dict containing the scrapy contracts enabled by default in Scrapy. You should +A dict containing the Scrapy contracts enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` instead. For more info see :ref:`topics-contracts`. @@ -1078,6 +1330,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. setting:: SPIDER_LOADER_WARN_ONLY + +SPIDER_LOADER_WARN_ONLY +----------------------- + +.. versionadded:: 1.3.3 + +Default: ``False`` + +By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +it will fail loudly if there is any ``ImportError`` exception. +But you can choose to silence this exception and turn it into a simple +warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. + +.. note:: + Some :ref:`scrapy commands ` run with this setting to ``True`` + already (i.e. they will only issue a warning and will not fail) + since they do not actually need to load spider classes to work: + :command:`scrapy runspider `, + :command:`scrapy settings `, + :command:`scrapy startproject `, + :command:`scrapy version `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES @@ -1187,6 +1462,101 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR + +TWISTED_REACTOR +--------------- + +.. versionadded:: 2.0 + +Default: ``None`` + +Import path of a given :mod:`~twisted.internet.reactor`. + +Scrapy will install this reactor if no other reactor is installed yet, such as +when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.CrawlerProcess` class. + +If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +need to install the correct reactor manually. You can do that using +:func:`~scrapy.utils.reactor.install_reactor`: + +.. autofunction:: scrapy.utils.reactor.install_reactor + +If a reactor is already installed, +:func:`~scrapy.utils.reactor.install_reactor` has no effect. + +:meth:`CrawlerRunner.__init__ ` raises +:exc:`Exception` if the installed reactor does not match the +:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:mod:`~twisted.internet.reactor` imports in project files and imported +third-party libraries will make Scrapy raise :exc:`Exception` when +it checks which reactor is installed. + +In order to use the reactor installed by Scrapy:: + + import scrapy + from twisted.internet import reactor + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + + +which raises :exc:`Exception`, becomes:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + from twisted.internet import reactor + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + + +The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which +means that Scrapy will not attempt to install any specific reactor, and the +default reactor defined by Twisted for the current platform will be used. This +is to maintain backward compatibility and avoid possible problems caused by +using a non-default reactor. + +For additional information, see :doc:`core/howto/choosing-reactor`. + .. setting:: URLLENGTH_LIMIT @@ -1198,16 +1568,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html .. setting:: USER_AGENT USER_AGENT ---------- -Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` -The default User-Agent to use when crawling, unless overridden. +The default User-Agent to use when crawling, unless overridden. This user agent is +also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` +if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and +there is no overridding User-Agent header specified for the request. Settings documented elsewhere: diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py index 19fc303b5..a37ffa0dc 100644 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -1,10 +1,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.auth` instead", + "use `scrapy.downloadermiddlewares.httpauth` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.utils.deprecate import create_deprecated_class -from scrapy.downloadermiddlewares.auth import AuthMiddleware - -HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) +from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py deleted file mode 100644 index 64239ab45..000000000 --- a/scrapy/downloadermiddlewares/auth.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -HTTP/FTP Authorization downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" -from six.moves.urllib.parse import unquote, urlunparse - -from w3lib.http import basic_auth_header - -from scrapy import signals -from scrapy.utils.httpobj import urlparse_cached - - -def credstrip_url(parsed_url): - """Strip username and password from an urlparse'd URL""" - return urlunparse(( - parsed_url.scheme, - parsed_url.netloc.split('@')[-1], - parsed_url.path, - parsed_url.params, - parsed_url.query, - parsed_url.fragment)) - - -def _unquote(s): - if s is not None: - return unquote(s) - - -class AuthMiddleware(object): - """ - Populate authorization credentials for HTTP and FTP requests. - - For http(s):// requests, set Basic HTTP Authorization header, - either from http_user and http_pass spider attributes, - or from URL netloc parsing. - - Also handle FTP credentials from ftp://user:password@... URLs, - populating request's meta accordingly. - """ - - @classmethod - def from_crawler(cls, crawler): - o = cls() - crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) - return o - - def spider_opened(self, spider): - usr = getattr(spider, 'http_user', '') - pwd = getattr(spider, 'http_pass', '') - if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - - def process_request(self, request, spider): - url = urlparse_cached(request) - if url.scheme.startswith('http'): - # do not override Auth header set priorly - if 'Authorization' in request.headers: - return - - new_url = None - # credentials from URL override spider attributes - if url.username or url.password: - auth = basic_auth_header(_unquote(url.username), - _unquote(url.password)) - - # no credentials in new url - new_url = credstrip_url(url) - - else: - auth = getattr(self, 'auth', None) - - if auth: - request.headers['Authorization'] = auth - if new_url: - return request.replace(url=new_url) - - elif url.scheme.startswith('ftp'): - if url.username or url.password: - # priorly set credentials take precedence - request.meta.setdefault('ftp_user', _unquote(url.username)) - request.meta.setdefault('ftp_password', _unquote(url.password)) - - # no credentials in new url - return request.replace(url=credstrip_url(url)) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 12b4372e5..7aa7a62bc 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,10 +1,31 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.auth` instead", - ScrapyDeprecationWarning) +""" +HTTP basic auth downloader middleware -from scrapy.utils.deprecate import create_deprecated_class -from scrapy.downloadermiddlewares.auth import AuthMiddleware +See documentation in docs/topics/downloader-middleware.rst +""" -HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) +from w3lib.http import basic_auth_header + +from scrapy import signals + + +class HttpAuthMiddleware(object): + """Set Basic HTTP Authorization header + (http_user and http_pass spider class attributes)""" + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o + + def spider_opened(self, spider): + usr = getattr(spider, 'http_user', '') + pwd = getattr(spider, 'http_pass', '') + if usr or pwd: + self.auth = basic_auth_header(usr, pwd) + + def process_request(self, request, spider): + auth = getattr(self, 'auth', None) + if auth and b'Authorization' not in request.headers: + request.headers[b'Authorization'] = auth diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py new file mode 100644 index 000000000..a7bff6239 --- /dev/null +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -0,0 +1,49 @@ +from urllib.parse import unquote, urlunparse + +from scrapy.utils.httpobj import urlparse_cached + + +class UriUserinfoMiddleware(object): + """Downloader middleware that replaces `URI userinfo`_ data (user credentials + for HTTP or FTP specified in the request URL) with the corresponding meta + keys for later middlewares or download handlers to use them for + authentication. + + It sets: + + - :reqmeta:`ftp_user` and :reqmeta:`ftp_password` for FTP requests + + - :reqmeta:`http_user` and :reqmeta:`http_pass` for HTTP and HTTPS + requests + + .. _URI userinfo: https://tools.ietf.org/html/rfc2396.html#section-3.2.2 + """ + + def process_request(self, request, spider): + url = urlparse_cached(request) + if url.username is None and url.password is None: + return + + if url.scheme.startswith('http'): + username_field, password_field = 'http_user', 'http_pass' + elif url.scheme.startswith('ftp'): + username_field, password_field = 'ftp_user', 'ftp_password' + else: + return + + for key, value in ((username_field, url.username), + (password_field, url.password)): + if value is not None: + request.meta.setdefault(key, unquote(value)) + + userinfoless_url = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc.split('@')[-1], + parsed_url.path, + parsed_url.params, + parsed_url.query, + parsed_url.fragment, + ) + ) + return request.replace(url=userinfoless_url) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..5ee6280a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,6 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py deleted file mode 100644 index 6f7a583ef..000000000 --- a/tests/test_downloadermiddleware_auth.py +++ /dev/null @@ -1,186 +0,0 @@ -import unittest - -from scrapy.http import Request -from scrapy.downloadermiddlewares.auth import AuthMiddleware -from scrapy.spiders import Spider - - -class TestSpider(Spider): - http_user = 'foo' - http_pass = 'bar' - - -class NoAuthTestSpider(Spider): - """A test spider that does not set http auth atttributes""" - - -class AuthMiddlewareNoAuthTest(unittest.TestCase): - - def setUp(self): - self.mw = AuthMiddleware() - self.spider = NoAuthTestSpider('bar') - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - def test_no_auth_http(self): - req = Request('http://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn('Authorization', req.headers) - - def test_no_auth_ftp(self): - req = Request('ftp://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn('ftp_user', req.meta) - self.assertNotIn('ftp_password', req.meta) - - -class AuthMiddlewareTest(unittest.TestCase): - - def setUp(self): - self.mw = AuthMiddleware() - self.spider = TestSpider('foo') - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - -class AuthMiddlewareHttpAuthTest(AuthMiddlewareTest): - - def test_auth(self): - req = Request('http://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') - - def test_auth_already_set(self): - req = Request('http://scrapytest.org/', - headers=dict(Authorization='Digest 123')) - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Digest 123') - - def test_auth_from_http_url(self): - req = Request('http://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_https_url(self): - req = Request('https://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'https://scrapytest.org/') - - def test_auth_from_http_url_no_spider_attrs(self): - class AnotherTestSpider(Spider): - pass - req = Request('http://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, AnotherTestSpider('bar')) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_https_url_no_spider_attrs(self): - class AnotherTestSpider(Spider): - pass - req = Request('https://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, AnotherTestSpider('bar')) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'https://scrapytest.org/') - - def test_auth_from_http_url_empty_pass(self): - req = Request('http://username:@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_http_url_pass_none(self): - req = Request('http://username@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6Tm9uZQ==') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_http_url_empty_user(self): - req = Request('http://:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic OnBhc3N3b3Jk') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - -class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): - - def test_no_auth_from_ftp_url_meta_unchanged(self): - usr, pwd = 'u', 'p' - req = Request('ftp://scrapytest.org/', - meta={"ftp_user": usr, "ftp_password": pwd}) - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.meta['ftp_user'], usr) - self.assertEquals(req.meta['ftp_password'], pwd) - - def test_auth_from_ftp_url_meta_unchanged(self): - """Request's meta credentials are kept as-is, - but URL is stripped from credentials - """ - usr, pwd = 'u', 'p' - req = Request('ftp://username:password@scrapytest.org/', - meta={"ftp_user": usr, "ftp_password": pwd}) - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.meta['ftp_user'], usr) - self.assertEquals(new_req.meta['ftp_password'], pwd) - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url(self): - req = Request('ftp://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_encoded_delims_user(self): - req = Request('ftp://username%3A:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username:') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_encoded_delims_password(self): - req = Request('ftp://username:pass%40word@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], 'pass@word') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_empty_user(self): - req = Request('ftp://:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], '') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_empty_pass(self): - req = Request('ftp://username:@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], '') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_pass_none(self): - req = Request('ftp://username@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], None) - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py new file mode 100644 index 000000000..425a5cc79 --- /dev/null +++ b/tests/test_downloadermiddleware_httpauth.py @@ -0,0 +1,32 @@ +import unittest + +from scrapy.http import Request +from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.spiders import Spider + + +class TestSpider(Spider): + http_user = 'foo' + http_pass = 'bar' + + +class HttpAuthMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpider('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + + def test_auth_already_set(self): + req = Request('http://scrapytest.org/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py new file mode 100644 index 000000000..13dcd4736 --- /dev/null +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -0,0 +1,72 @@ +import unittest + +from scrapy.http import Request +from scrapy.downloadermiddlewares.uriuserinfo import UriUserinfoMiddleware +from scrapy.spiders import Spider + + +class BaseTestCase: + + class Implementation(unittest.TestCase): + + def setUp(self): + self.mw = UriUserinfoMiddleware() + self.spider = Spider('bar') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_username_and_password(self): + req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertEqual(req.meta[self.password_field], 'bar') + + def test_username_and_empty_password(self): + req = Request('{}://foo:@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertEqual(req.meta[self.password_field], '') + + def test_username_and_no_password(self): + req = Request('{}://foo@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertNotIn(self.password_field, req.meta) + + def test_empty_username_and_nonempty_password(self): + req = Request('{}://:bar@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], '') + self.assertEqual(req.meta[self.password_field], 'bar') + + def test_no_username_and_no_password(self): + req = Request('{}://scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn(self.username_field, req.meta) + self.assertNotIn(self.password_field, req.meta) + + def test_unquoting(self): + req = Request('{}://foo%3A:b%40r@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo:') + self.assertEqual(req.meta[self.password_field], 'b@r') + + +class UriUserinfoMiddlewareFTPTest(BaseTestCase.Implementation): + protocol = 'ftp' + username_field = 'ftp_user' + password_field = 'ftp_password' + + +class UriUserinfoMiddlewareHTTPTest(BaseTestCase.Implementation): + protocol = 'http' + username_field = 'http_user' + password_field = 'http_pass' + + +class UriUserinfoMiddlewareHTTPSTest(BaseTestCase.Implementation): + protocol = 'https' + username_field = 'http_user' + password_field = 'http_pass' From 6a228241411465b43277ecde545afeb011a31cd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 6 Apr 2020 02:36:21 +0200 Subject: [PATCH 14/29] Add http_user and http_pass to the list of Request.meta keys --- docs/topics/request-response.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b2a60ff39..52b510675 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -340,8 +340,10 @@ Those are: * :reqmeta:`download_latency` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`proxy` -* ``ftp_user`` (See :setting:`FTP_USER` for more info) -* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`http_user` +* :reqmeta:`http_pass` +* :reqmeta:`ftp_user` +* :reqmeta:`ftp_password` * :reqmeta:`referrer_policy` * :reqmeta:`max_retry_times` From 54bec894fc136a53ebab33f020a4a2ee5121cccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:27:22 +0200 Subject: [PATCH 15/29] UriUserinfoMiddleware: fix variable name --- scrapy/downloadermiddlewares/uriuserinfo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py index a7bff6239..e16657dab 100644 --- a/scrapy/downloadermiddlewares/uriuserinfo.py +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -20,19 +20,19 @@ class UriUserinfoMiddleware(object): """ def process_request(self, request, spider): - url = urlparse_cached(request) - if url.username is None and url.password is None: + parsed_url = urlparse_cached(request) + if parsed_url.username is None and parsed_url.password is None: return - if url.scheme.startswith('http'): + if parsed_url.scheme.startswith('http'): username_field, password_field = 'http_user', 'http_pass' - elif url.scheme.startswith('ftp'): + elif parsed_url.scheme.startswith('ftp'): username_field, password_field = 'ftp_user', 'ftp_password' else: return - for key, value in ((username_field, url.username), - (password_field, url.password)): + for key, value in ((username_field, parsed_url.username), + (password_field, parsed_url.password)): if value is not None: request.meta.setdefault(key, unquote(value)) From b1802d6f089c86c833ad584651eb88ba0c559159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:28:14 +0200 Subject: [PATCH 16/29] tests/test_downloadermiddleware_uriuserinfo.py: mild code style change --- tests/test_downloadermiddleware_uriuserinfo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 13dcd4736..afb461515 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -48,7 +48,9 @@ class BaseTestCase: self.assertNotIn(self.password_field, req.meta) def test_unquoting(self): - req = Request('{}://foo%3A:b%40r@scrapytest.org/'.format(self.protocol)) + req = Request( + '{}://foo%3A:b%40r@scrapytest.org/'.format(self.protocol) + ) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.meta[self.username_field], 'foo:') self.assertEqual(req.meta[self.password_field], 'b@r') From 8218efaf46e87026b70ffd5b08f1ce25e7120fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:29:49 +0200 Subject: [PATCH 17/29] test_downloadermiddleware_uriuserinfo.py: fix tests --- tests/test_downloadermiddleware_uriuserinfo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index afb461515..42805865e 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -11,8 +11,6 @@ class BaseTestCase: def setUp(self): self.mw = UriUserinfoMiddleware() - self.spider = Spider('bar') - self.mw.spider_opened(self.spider) def tearDown(self): del self.mw From 3e70c7a182b4460d1b139d2fb6741a1902c07263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:41:07 +0200 Subject: [PATCH 18/29] HttpAuthMiddleware: implement meta support --- scrapy/downloadermiddlewares/httpauth.py | 12 ++++++++++-- tests/test_downloadermiddleware_httpauth.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..ee6c35619 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -26,6 +26,14 @@ class HttpAuthMiddleware: self.auth = basic_auth_header(usr, pwd) def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and b'Authorization' not in request.headers: + if b'Authorization' in request.headers: + return + + usr = request.meta.get('http_user', '') + pwd = request.meta.get('http_pass', '') + if usr or pwd: + auth = basic_auth_header(usr, pwd) + else: + auth = getattr(self, 'auth', None) + if auth: request.headers[b'Authorization'] = auth diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..91e4ef23b 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -30,3 +30,23 @@ class HttpAuthMiddlewareTest(unittest.TestCase): headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') + + def test_auth_already_set_with_meta(self): + meta = {'http_user': 'bar', 'http_pass': 'foo'} + req = Request('http://scrapytest.org/', + headers=dict(Authorization='Digest 123'), + meta=meta) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + def test_auth_meta(self): + meta = {'http_user': 'bar', 'http_pass': 'foo'} + req = Request('http://scrapytest.org/', meta=meta) + assert self.mw.process_request(req, Spider('bar')) is None + self.assertEqual(req.headers['Authorization'], b'Basic YmFyOmZvbw==') + + def test_auth_meta_override(self): + meta = {'http_user': 'bar', 'http_pass': 'foo'} + req = Request('http://scrapytest.org/', meta=meta) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Basic YmFyOmZvbw==') From 0a41ed2cea65ddff2955342f76152ef94eead155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:45:58 +0200 Subject: [PATCH 19/29] test_downloadermiddleware_uriuserinfo.py: remove unused import --- tests/test_downloadermiddleware_uriuserinfo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 42805865e..f7d2c47c2 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -2,7 +2,6 @@ import unittest from scrapy.http import Request from scrapy.downloadermiddlewares.uriuserinfo import UriUserinfoMiddleware -from scrapy.spiders import Spider class BaseTestCase: From 9e6dd16ca3c24e8bce5ff68049a4fb428dabbd56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:49:16 +0200 Subject: [PATCH 20/29] test_downloadermiddleware_uriuserinfo: restore needed instance variable --- tests/test_downloadermiddleware_uriuserinfo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index f7d2c47c2..4b430aeb8 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -2,6 +2,7 @@ import unittest from scrapy.http import Request from scrapy.downloadermiddlewares.uriuserinfo import UriUserinfoMiddleware +from scrapy.spiders import Spider class BaseTestCase: @@ -10,6 +11,7 @@ class BaseTestCase: def setUp(self): self.mw = UriUserinfoMiddleware() + self.spider = Spider('bar') def tearDown(self): del self.mw From a0bcdd7d4ddb04e0680093a8d1ba13db301a6852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 12:52:57 +0200 Subject: [PATCH 21/29] test_downloadermiddleware_uriuserinfo: fix tests --- .../test_downloadermiddleware_uriuserinfo.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 4b430aeb8..5a0c12506 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -18,25 +18,33 @@ class BaseTestCase: def test_username_and_password(self): req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) - assert self.mw.process_request(req, self.spider) is None + userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request.url == userinfoless_url self.assertEqual(req.meta[self.username_field], 'foo') self.assertEqual(req.meta[self.password_field], 'bar') def test_username_and_empty_password(self): req = Request('{}://foo:@scrapytest.org/'.format(self.protocol)) - assert self.mw.process_request(req, self.spider) is None + userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request.url == userinfoless_url self.assertEqual(req.meta[self.username_field], 'foo') self.assertEqual(req.meta[self.password_field], '') def test_username_and_no_password(self): req = Request('{}://foo@scrapytest.org/'.format(self.protocol)) - assert self.mw.process_request(req, self.spider) is None + userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request.url == userinfoless_url self.assertEqual(req.meta[self.username_field], 'foo') self.assertNotIn(self.password_field, req.meta) def test_empty_username_and_nonempty_password(self): req = Request('{}://:bar@scrapytest.org/'.format(self.protocol)) - assert self.mw.process_request(req, self.spider) is None + userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request.url == userinfoless_url self.assertEqual(req.meta[self.username_field], '') self.assertEqual(req.meta[self.password_field], 'bar') @@ -50,7 +58,9 @@ class BaseTestCase: req = Request( '{}://foo%3A:b%40r@scrapytest.org/'.format(self.protocol) ) - assert self.mw.process_request(req, self.spider) is None + userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request.url == userinfoless_url self.assertEqual(req.meta[self.username_field], 'foo:') self.assertEqual(req.meta[self.password_field], 'b@r') From 98207f8f168ca129d5a77f5f5ec644edacf9fdcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 16:03:47 +0200 Subject: [PATCH 22/29] Update scrapy/downloadermiddlewares/uriuserinfo.py Co-Authored-By: Eugenio Lacuesta --- scrapy/downloadermiddlewares/uriuserinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py index e16657dab..7eabe15d6 100644 --- a/scrapy/downloadermiddlewares/uriuserinfo.py +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -3,7 +3,7 @@ from urllib.parse import unquote, urlunparse from scrapy.utils.httpobj import urlparse_cached -class UriUserinfoMiddleware(object): +class UriUserinfoMiddleware: """Downloader middleware that replaces `URI userinfo`_ data (user credentials for HTTP or FTP specified in the request URL) with the corresponding meta keys for later middlewares or download handlers to use them for From 0bd1918ac8f0a0e604a0c3f070ec860826fec50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Apr 2020 16:07:05 +0200 Subject: [PATCH 23/29] =?UTF-8?q?UriUserinfo=20=E2=86=92=20UriUserInfo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/downloader-middleware.rst | 6 +++--- docs/topics/settings.rst | 6 +++--- scrapy/downloadermiddlewares/uriuserinfo.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_downloadermiddleware_uriuserinfo.py | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b9b81aa54..fc88ac03d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -332,7 +332,7 @@ HttpAuthMiddleware You can alternatively specify ``http_user`` and ``http_pass`` in :attr:`Request.meta `, or use - :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`. + :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware`. .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -1140,13 +1140,13 @@ DownloaderStats setting. -UriUserinfoMiddleware +UriUserInfoMiddleware --------------------- .. module:: scrapy.downloadermiddlewares.uriuserinfo :synopsis: URI Userinfo Middleware -.. autoclass:: UriUserinfoMiddleware +.. autoclass:: UriUserInfoMiddleware UserAgentMiddleware diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 611009680..21bf0969f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -534,7 +534,7 @@ Default:: { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware': 200, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, @@ -824,7 +824,7 @@ It can be overriden in a request in any of the following ways: - Specifying ``ftp_password`` in :attr:`Request.meta ` - Specifying the password in :attr:`Request.url ` - (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware`) .. note:: Paraphrasing `RFC 1635`_, although it is common to use either the password @@ -849,7 +849,7 @@ It can be overriden in a request in any of the following ways: - Specifying ``ftp_user`` in :attr:`Request.meta ` - Specifying the username in :attr:`Request.url ` - (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware`) .. setting:: ITEM_PIPELINES diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py index 7eabe15d6..8c9f64f36 100644 --- a/scrapy/downloadermiddlewares/uriuserinfo.py +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -3,7 +3,7 @@ from urllib.parse import unquote, urlunparse from scrapy.utils.httpobj import urlparse_cached -class UriUserinfoMiddleware: +class UriUserInfoMiddleware: """Downloader middleware that replaces `URI userinfo`_ data (user credentials for HTTP or FTP specified in the request URL) with the corresponding meta keys for later middlewares or download handlers to use them for diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 96c4555cf..736a121aa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -94,7 +94,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware': 200, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 5a0c12506..2cdc3312d 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.downloadermiddlewares.uriuserinfo import UriUserinfoMiddleware +from scrapy.downloadermiddlewares.uriuserinfo import UriUserInfoMiddleware from scrapy.spiders import Spider @@ -10,7 +10,7 @@ class BaseTestCase: class Implementation(unittest.TestCase): def setUp(self): - self.mw = UriUserinfoMiddleware() + self.mw = UriUserInfoMiddleware() self.spider = Spider('bar') def tearDown(self): @@ -65,19 +65,19 @@ class BaseTestCase: self.assertEqual(req.meta[self.password_field], 'b@r') -class UriUserinfoMiddlewareFTPTest(BaseTestCase.Implementation): +class UriUserInfoMiddlewareFTPTest(BaseTestCase.Implementation): protocol = 'ftp' username_field = 'ftp_user' password_field = 'ftp_password' -class UriUserinfoMiddlewareHTTPTest(BaseTestCase.Implementation): +class UriUserInfoMiddlewareHTTPTest(BaseTestCase.Implementation): protocol = 'http' username_field = 'http_user' password_field = 'http_pass' -class UriUserinfoMiddlewareHTTPSTest(BaseTestCase.Implementation): +class UriUserInfoMiddlewareHTTPSTest(BaseTestCase.Implementation): protocol = 'https' username_field = 'http_user' password_field = 'http_pass' From 924a95d2f55acb1794b0ce1536f3d234c8e51aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 19:04:34 +0200 Subject: [PATCH 24/29] test_downloadermiddleware_uriuserinfo.py: cover unhandled protocols --- .../test_downloadermiddleware_uriuserinfo.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 2cdc3312d..5807e3edb 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -5,9 +5,11 @@ from scrapy.downloadermiddlewares.uriuserinfo import UriUserInfoMiddleware from scrapy.spiders import Spider -class BaseTestCase: +class AbstractWrapper: + """Class to define base test case classes that can be inherited but are not + executed themselves.""" - class Implementation(unittest.TestCase): + class BaseTestCase(unittest.TestCase): def setUp(self): self.mw = UriUserInfoMiddleware() @@ -16,6 +18,8 @@ class BaseTestCase: def tearDown(self): del self.mw + class ProtocolTestCase(BaseTestCase): + def test_username_and_password(self): req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) userinfoless_url = '{}://scrapytest.org/'.format(self.protocol) @@ -65,19 +69,29 @@ class BaseTestCase: self.assertEqual(req.meta[self.password_field], 'b@r') -class UriUserInfoMiddlewareFTPTest(BaseTestCase.Implementation): +class FTPTest(AbstractWrapper.ProtocolTestCase): protocol = 'ftp' username_field = 'ftp_user' password_field = 'ftp_password' -class UriUserInfoMiddlewareHTTPTest(BaseTestCase.Implementation): +class HTTPTest(AbstractWrapper.ProtocolTestCase): protocol = 'http' username_field = 'http_user' password_field = 'http_pass' -class UriUserInfoMiddlewareHTTPSTest(BaseTestCase.Implementation): +class HTTPSTest(AbstractWrapper.ProtocolTestCase): protocol = 'https' username_field = 'http_user' password_field = 'http_pass' + + +class UnhandledProtocolTest(AbstractWrapper.BaseTestCase): + protocol = 's3' + + def test_unhandled_protocol(self): + req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) + processed_request = self.mw.process_request(req, self.spider) + assert processed_request == None + assert not req.meta From 0166356b83d43043f008b73413954f0c912a9167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Apr 2020 13:48:36 +0200 Subject: [PATCH 25/29] Fix Flake8-reported issues --- tests/test_downloadermiddleware_uriuserinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 5807e3edb..5a14167af 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -93,5 +93,5 @@ class UnhandledProtocolTest(AbstractWrapper.BaseTestCase): def test_unhandled_protocol(self): req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) processed_request = self.mw.process_request(req, self.spider) - assert processed_request == None + assert processed_request is None assert not req.meta From 43f4f0f7d6de7af925a4edf7632fe4f6fcea11da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 09:43:38 +0100 Subject: [PATCH 26/29] =?UTF-8?q?scrapytest.org=20=E2=86=92=20example.com?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_downloadermiddleware_httpauth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index f6f2caaef..e9e610d5a 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -86,7 +86,7 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): def test_auth_already_set_with_meta(self): meta = {"http_user": "bar", "http_pass": "foo"} req = Request( - "http://scrapytest.org/", + "http://example.com/", headers=dict(Authorization="Digest 123"), meta=meta, ) @@ -95,12 +95,12 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): def test_auth_meta(self): meta = {"http_user": "bar", "http_pass": "foo"} - req = Request("http://scrapytest.org/", meta=meta) + req = Request("http://example.com/", meta=meta) assert self.mw.process_request(req, Spider("bar")) is None self.assertEqual(req.headers["Authorization"], b"Basic YmFyOmZvbw==") def test_auth_meta_override(self): meta = {"http_user": "bar", "http_pass": "foo"} - req = Request("http://scrapytest.org/", meta=meta) + req = Request("http://example.com/", meta=meta) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Basic YmFyOmZvbw==") From f0dac4cbf8c60ea8360767fcc764bec60455b7c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 09:46:24 +0100 Subject: [PATCH 27/29] Add explicit return statements --- scrapy/downloadermiddlewares/httpauth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 97194fa66..9004a887b 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -45,7 +45,7 @@ class HttpAuthMiddleware: self, request: Request, spider: Spider ) -> Union[Request, Response, None]: if b"Authorization" in request.headers: - return + return None usr = request.meta.get("http_user", "") pwd = request.meta.get("http_pass", "") if usr or pwd: @@ -56,3 +56,4 @@ class HttpAuthMiddleware: auth = None if auth: request.headers[b"Authorization"] = auth + return None From cb88457f95620adcff8b8ea47733d27b47e22287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 10 Jan 2025 11:32:12 +0100 Subject: [PATCH 28/29] Run pre-commit --- scrapy/downloadermiddlewares/uriuserinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py index 6deb5b0eb..ba7ee0f1d 100644 --- a/scrapy/downloadermiddlewares/uriuserinfo.py +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -22,14 +22,14 @@ class UriUserInfoMiddleware: def process_request(self, request, spider): parsed_url = urlparse_cached(request) if parsed_url.username is None and parsed_url.password is None: - return + return None if parsed_url.scheme.startswith("http"): username_field, password_field = "http_user", "http_pass" elif parsed_url.scheme.startswith("ftp"): username_field, password_field = "ftp_user", "ftp_password" else: - return + return None for key, value in ( (username_field, parsed_url.username), From 0fe0109386df7a0ee4cc3e7f61459c1fe6a6e603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 10 Jan 2025 14:52:26 +0100 Subject: [PATCH 29/29] Complete test coverage, support explicit default ports --- scrapy/downloadermiddlewares/httpauth.py | 20 +- tests/test_downloadermiddleware_httpauth.py | 255 +++++++++++------- .../test_downloadermiddleware_uriuserinfo.py | 3 - 3 files changed, 181 insertions(+), 97 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index da370977d..2c8649052 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -22,9 +22,22 @@ if TYPE_CHECKING: from scrapy.http import Response +_NO_PORT = object() +_DEFAULT_PORTS = { + "http": 80, + "https": 443, +} + + def _origin(request: Request) -> str: parsed_url = urlparse_cached(request) - return f"{parsed_url.scheme}://{parsed_url.netloc}" + scheme = parsed_url.scheme + netloc = ( + parsed_url.netloc + if parsed_url.port != _DEFAULT_PORTS[scheme] + else parsed_url.hostname + ) + return f"{scheme}://{netloc}" def _setdefault_auth_origin(request: Request) -> str: @@ -59,7 +72,10 @@ class HttpAuthMiddleware: def process_request( self, request: Request, spider: Spider ) -> Request | Response | None: - if b"Authorization" in request.headers: + if ( + b"Authorization" in request.headers + or urlparse_cached(request).scheme not in _DEFAULT_PORTS + ): return None user = request.meta.get("http_user", "") password = request.meta.get("http_pass", "") diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 6f8e6f7bb..f17a2573f 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,106 +1,177 @@ -import unittest - -from w3lib.http import basic_auth_header +import pytest from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.http import Request from scrapy.spiders import Spider -class TestSpiderLegacy(Spider): - http_user = "foo" - http_pass = "bar" +@pytest.mark.parametrize( + ("config", "expected"), + ( + # Baseline + ({}, None), + # Spider attributes. + # http_auth_domain=None allows any domain. + ( + {"spider_attributes": {"http_user": "su", "http_auth_domain": None}}, + b"Basic c3U6", + ), + ( + {"spider_attributes": {"http_pass": "sp", "http_auth_domain": None}}, + b"Basic OnNw", + ), + ( + { + "spider_attributes": { + "http_user": "su", + "http_pass": "sp", + "http_auth_domain": None, + } + }, + b"Basic c3U6c3A=", + ), + # http_auth_domain=domain allows only that domain and subdomains. + ( + {"spider_attributes": {"http_user": "su", "http_auth_domain": "a.example"}}, + b"Basic c3U6", + ), + ( + { + "url": "https://s.a.example/a", + "spider_attributes": { + "http_user": "su", + "http_auth_domain": "a.example", + }, + }, + b"Basic c3U6", + ), + ( + {"spider_attributes": {"http_user": "su", "http_auth_domain": "b.example"}}, + None, + ), + # http_auth_domain must be defined if http_user or http_pass are. + ({"spider_attributes": {"http_user": "su"}}, AttributeError), + # Request.meta. + ({"meta": {"http_user": "mu"}}, b"Basic bXU6"), + ({"meta": {"http_pass": "mp"}}, b"Basic Om1w"), + ({"meta": {"http_user": "mu", "http_pass": "mp"}}, b"Basic bXU6bXA="), + # Request.meta["auth_origin"]=origin prevents other origins. + # + # Note: auth_origin is not meant to be set by users, it is set the + # first time a request is processed by the middleware. See + # test_origin_setdefault. + ( + {"meta": {"auth_origin": "https://a.example", "http_user": "mu"}}, + b"Basic bXU6", + ), + ( + { + "url": "https://a.example:443", + "meta": {"auth_origin": "https://a.example", "http_user": "mu"}, + }, + b"Basic bXU6", + ), + ( + { + "url": "https://s.a.example", + "meta": {"auth_origin": "https://a.example", "http_user": "mu"}, + }, + None, + ), + ({"meta": {"auth_origin": "http://a.example", "http_user": "mu"}}, None), + ({"meta": {"auth_origin": "https://a.example:1", "http_user": "mu"}}, None), + ({"meta": {"auth_origin": "https://b.example", "http_user": "mu"}}, None), + # Takes priority over spider attributes. + ( + { + "meta": {"http_user": "mu"}, + "spider_attributes": { + "http_user": "su", + "http_pass": "sp", + "http_auth_domain": None, + }, + }, + b"Basic bXU6", + ), + # If the Authorization header is set, it is not modified. + ( + { + "headers": {"Authorization": "a"}, + "spider_attributes": {"http_user": "su", "http_auth_domain": None}, + }, + b"a", + ), + ({"headers": {"Authorization": "a"}, "meta": {"http_user": "mu"}}, b"a"), + # If a non-HTTP request is received, nothing is done. + ( + { + "url": "ftp://example.com", + "spider_attributes": {"http_user": "su", "http_auth_domain": None}, + }, + None, + ), + ({"url": "s3://example.com", "meta": {"http_user": "mu"}}, None), + ), +) +def test_main(config, expected): + url = config.get("url", "https://a.example") + headers = config.get("headers", {}) + meta = config.get("meta", {}) + spider_attributes = config.get("spider_attributes", {}) + class TestSpider(Spider): + pass -class TestSpider(Spider): - http_user = "foo" - http_pass = "bar" - http_auth_domain = "example.com" + for k, v in spider_attributes.items(): + setattr(TestSpider, k, v) + mw = HttpAuthMiddleware() + spider = TestSpider("foo") -class TestSpiderAny(Spider): - http_user = "foo" - http_pass = "bar" - http_auth_domain = None + if isinstance(expected, type) and issubclass(expected, Exception): + with pytest.raises(expected): + mw.spider_opened(spider) + return - -class HttpAuthMiddlewareLegacyTest(unittest.TestCase): - def setUp(self): - self.spider = TestSpiderLegacy("foo") - - def test_auth(self): - with self.assertRaises(AttributeError): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) - - -class HttpAuthMiddlewareTest(unittest.TestCase): - def setUp(self): - self.mw = HttpAuthMiddleware() - self.spider = TestSpider("foo") - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - def test_no_auth(self): - req = Request("http://noauth.example/") - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) - - def test_auth_domain(self): - req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - def test_auth_subdomain(self): - req = Request("http://foo.example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - def test_auth_already_set(self): - req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - - -class HttpAuthAnyMiddlewareTest(unittest.TestCase): - def setUp(self): - self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny("foo") - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - def test_auth(self): - req = Request("http://example.com/") - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - def test_auth_already_set(self): - req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - - def test_auth_already_set_with_meta(self): - meta = {"http_user": "bar", "http_pass": "foo"} - req = Request( - "http://example.com/", - headers={"Authorization": "Digest 123"}, - meta=meta, + mw.spider_opened(spider) + request = Request(url, headers=headers, meta=meta) + assert mw.process_request(request, spider) is None + if expected is None: + assert "Authorization" not in request.headers + else: + assert request.headers["Authorization"] == expected, repr( + request.headers["Authorization"] ) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - def test_auth_meta(self): - meta = {"http_user": "bar", "http_pass": "foo"} - req = Request("http://example.com/", meta=meta) - assert self.mw.process_request(req, Spider("bar")) is None - self.assertEqual(req.headers["Authorization"], b"Basic YmFyOmZvbw==") - def test_auth_meta_override(self): - meta = {"http_user": "bar", "http_pass": "foo"} - req = Request("http://example.com/", meta=meta) - assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Basic YmFyOmZvbw==") +@pytest.mark.parametrize( + ("meta", "url", "output_value"), + ( + ({}, "https://example.com/a", None), + ({"http_user": "a", "auth_origin": "foo"}, "https://example.com/a", "foo"), + ({"http_user": "a"}, "https://example.com/a", "https://example.com"), + ({"http_user": "a"}, "http://example.com/a", "http://example.com"), + ({"http_user": "a"}, "https://example.com:443/a", "https://example.com"), + ({"http_user": "a"}, "http://example.com:80/a", "http://example.com"), + ({"http_user": "a"}, "https://example.com:80/a", "https://example.com:80"), + ({"http_user": "a"}, "http://example.com:443/a", "http://example.com:443"), + ({"http_user": "a"}, "https://example.com:1234/a", "https://example.com:1234"), + ({"http_user": "a"}, "http://example.com:1234/a", "http://example.com:1234"), + ), +) +def test_origin_setdefault(meta, url, output_value): + """When request.meta is used for authorization, an auth_origin meta key is + defined on the request if not defined already.""" + + class TestSpider(Spider): + pass + + mw = HttpAuthMiddleware() + spider = TestSpider("foo") + mw.spider_opened(spider) + request = Request(url, meta=meta) + assert mw.process_request(request, spider) is None + if output_value is None: + assert "auth_origin" not in request.meta + else: + assert request.meta["auth_origin"] == output_value diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py index 62bca7ceb..27f6f507e 100644 --- a/tests/test_downloadermiddleware_uriuserinfo.py +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -14,9 +14,6 @@ class AbstractWrapper: self.mw = UriUserInfoMiddleware() self.spider = Spider("bar") - def tearDown(self): - del self.mw - class ProtocolTestCase(BaseTestCase): def test_username_and_password(self): req = Request(f"{self.protocol}://foo:bar@example.com/")