From 9fc45cccb538366d1da123dbfdc2a29e3e7d7631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 6 Apr 2020 02:17:11 +0200 Subject: [PATCH] Implement UriUserinfoMiddleware --- docs/faq.rst | 2 +- docs/topics/downloader-middleware.rst | 543 +++++++++++++----- docs/topics/settings.rst | 493 ++++++++++++++-- .../contrib/downloadermiddleware/httpauth.py | 7 +- scrapy/downloadermiddlewares/auth.py | 85 --- scrapy/downloadermiddlewares/httpauth.py | 37 +- scrapy/downloadermiddlewares/uriuserinfo.py | 49 ++ scrapy/settings/default_settings.py | 1 + tests/test_downloadermiddleware_auth.py | 186 ------ tests/test_downloadermiddleware_httpauth.py | 32 ++ .../test_downloadermiddleware_uriuserinfo.py | 72 +++ 11 files changed, 1006 insertions(+), 501 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/auth.py create mode 100644 scrapy/downloadermiddlewares/uriuserinfo.py delete mode 100644 tests/test_downloadermiddleware_auth.py create mode 100644 tests/test_downloadermiddleware_httpauth.py create mode 100644 tests/test_downloadermiddleware_uriuserinfo.py diff --git a/docs/faq.rst b/docs/faq.rst index d3efc09fc..415331515 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ See previous question. Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- -Yes, see :class:`~scrapy.downloadermiddlewares.auth.AuthMiddleware`. +Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`. Why does Scrapy download pages in English instead of my native language? ------------------------------------------------------------------------ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 01927e584..b9b81aa54 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it -in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None` +in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { @@ -52,11 +52,17 @@ as its value. For example, if you want to disable the user-agent middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _topics-downloader-middleware-custom: + Writing your own downloader middleware ====================================== -Each middleware component is a Python class that defines one or -more of the following methods: +Each downloader middleware is a Python class that defines one or more of the +methods defined below. + +The main entry point is the ``from_crawler`` class method, which receives a +:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` +object gives you access, for example, to the :ref:`settings `. .. module:: scrapy.downloadermiddlewares @@ -157,6 +163,17 @@ more of the following methods: :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -182,7 +199,7 @@ CookiesMiddleware This middleware enables working with sites that require cookies, such as those that use sessions. It keeps track of cookies sent by web servers, and - send them back on subsequent requests (from that spider), just like web + sends them back on subsequent requests (from that spider), just like web browsers do. The following settings can be used to configure the cookie middleware: @@ -226,6 +243,15 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. + +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. + .. setting:: COOKIES_DEBUG COOKIES_DEBUG @@ -233,19 +259,19 @@ COOKIES_DEBUG Default: ``False`` -If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie`` -header) and all cookies received in responses (ie. ``Set-Cookie`` header). +If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie`` +header) and all cookies received in responses (i.e. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - 2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ Set-Cookie: ip_isocode=US Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) (referer: None) + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [...] @@ -278,43 +304,6 @@ DownloadTimeoutMiddleware :reqmeta:`download_timeout` Request.meta key; this is supported even when DownloadTimeoutMiddleware is disabled. -AuthMiddleware ------------------- - -.. module:: scrapy.downloadermiddlewares.auth - :synopsis: HTTP/FTP Auth downloader middleware - -.. class:: AuthMiddleware - - This middleware populates authentication credentials for HTTP and FTP requests. - - To enable HTTP authentication (`Basic access authentication`_, aka. HTTP auth), - you have two options: - - - either set the ``http_user`` and ``http_pass`` attributes of the spider(s) - for which you need HTTP auth, - and these will be applied to all http(s):// requests, - - or, on a per-Request basis, include credentials using the standard URL - syntax of ``http://username:password@www.example.com/index.html`` - - To populate credentials for FTP requests, use URLs in the form of - ``ftp://user:password@www.example.com/document.txt``, which will - set the ``meta`` dict ``ftp_user`` and ``ftp_password`` values. - - Example of enabling HTTP Basic Auth for all HTTP requests:: - - from scrapy.spiders import CrawlSpider - - class SomeIntranetSiteSpider(CrawlSpider): - - http_user = 'someuser' - http_pass = 'somepass' - name = 'intranet.example.com' - - # .. rest of the spider code omitted ... - -.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication - HttpAuthMiddleware ------------------ @@ -323,7 +312,30 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware is deprecated and redirects to :class:`~.AuthMiddleware`. + This middleware authenticates all requests generated from certain spiders + using `Basic access authentication`_ (aka. HTTP auth). + + To enable HTTP authentication from certain spiders, set the ``http_user`` + and ``http_pass`` attributes of those spiders. + + Example:: + + from scrapy.spiders import CrawlSpider + + class SomeIntranetSiteSpider(CrawlSpider): + name = 'intranet.example.com' + http_user = 'someuser' + http_pass = 'somepass' + + .. reqmeta:: http_user + .. reqmeta:: http_pass + + You can alternatively specify ``http_user`` and ``http_pass`` in + :attr:`Request.meta `, or use + :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`. + +.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication + HttpCacheMiddleware ------------------- @@ -336,13 +348,13 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with two HTTP cache storage backends: + Scrapy ships with three HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` - setting. Or you can also implement your own storage backend. + setting. Or you can also :ref:`implement your own storage backend. ` Scrapy ships with two HTTP cache policies: @@ -354,26 +366,27 @@ HttpCacheMiddleware .. reqmeta:: dont_cache - You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`. + You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``. + +.. module:: scrapy.extensions.httpcache + :noindex: .. _httpcache-policy-dummy: Dummy policy (default) ~~~~~~~~~~~~~~~~~~~~~~ -This policy has no awareness of any HTTP Cache-Control directives. -Every request and its corresponding response are cached. When the same -request is seen again, the response is returned without transferring -anything from the Internet. +.. class:: DummyPolicy -The Dummy policy is useful for testing spiders faster (without having -to wait for downloads every time) and for trying your spider offline, -when an Internet connection is not available. The goal is to be able to -"replay" a spider run *exactly as it ran before*. + This policy has no awareness of any HTTP Cache-Control directives. + Every request and its corresponding response are cached. When the same + request is seen again, the response is returned without transferring + anything from the Internet. -In order to use this policy, set: - -* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy`` + The Dummy policy is useful for testing spiders faster (without having + to wait for downloads every time) and for trying your spider offline, + when an Internet connection is not available. The goal is to be able to + "replay" a spider run *exactly as it ran before*. .. _httpcache-policy-rfc2616: @@ -381,45 +394,44 @@ In order to use this policy, set: RFC2616 policy ~~~~~~~~~~~~~~ -This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP -Cache-Control awareness, aimed at production and used in continuous -runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). +.. class:: RFC2616Policy -what is implemented: + This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP + Cache-Control awareness, aimed at production and used in continuous + runs to avoid downloading unmodified data (to save bandwidth and speed up + crawls). -* Do not attempt to store responses/requests with `no-store` cache-control directive set -* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses -* Compute freshness lifetime from `max-age` cache-control directive -* Compute freshness lifetime from `Expires` response header -* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox) -* Compute current age from `Age` response header -* Compute current age from `Date` header -* Revalidate stale responses based on `Last-Modified` response header -* Revalidate stale responses based on `ETag` response header -* Set `Date` header for any received response missing it -* Support `max-stale` cache-control directive in requests + What is implemented: - This allows spiders to be configured with the full RFC2616 cache policy, - but avoid revalidation on a request-by-request basis, while remaining - conformant with the HTTP spec. + * Do not attempt to store responses/requests with ``no-store`` cache-control directive set + * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses + * Compute freshness lifetime from ``max-age`` cache-control directive + * Compute freshness lifetime from ``Expires`` response header + * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox) + * Compute current age from ``Age`` response header + * Compute current age from ``Date`` header + * Revalidate stale responses based on ``Last-Modified`` response header + * Revalidate stale responses based on ``ETag`` response header + * Set ``Date`` header for any received response missing it + * Support ``max-stale`` cache-control directive in requests - Example: + This allows spiders to be configured with the full RFC2616 cache policy, + but avoid revalidation on a request-by-request basis, while remaining + conformant with the HTTP spec. - Add `Cache-Control: max-stale=600` to Request headers to accept responses that - have exceeded their expiration time by no more than 600 seconds. + Example: - See also: RFC2616, 14.9.3 + Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that + have exceeded their expiration time by no more than 600 seconds. -what is missing: + See also: RFC2616, 14.9.3 -* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 -* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 -* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 -* ... probably others .. + What is missing: -In order to use this policy, set: - -* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy`` + * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 + * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 + * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10 + * ... probably others .. .. _httpcache-storage-fs: @@ -427,67 +439,102 @@ In order to use this policy, set: Filesystem storage backend (default) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -File system storage backend is available for the HTTP cache middleware. +.. class:: FilesystemCacheStorage -In order to use this storage backend, set: + File system storage backend is available for the HTTP cache middleware. -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage`` + Each request/response pair is stored in a different directory containing + the following files: -Each request/response pair is stored in a different directory containing -the following files: + * ``request_body`` - the plain request body - * ``request_body`` - the plain request body - * ``request_headers`` - the request headers (in raw HTTP format) - * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) - * ``meta`` - some metadata of this cache resource in Python ``repr()`` format - (grep-friendly format) - * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more - efficient deserialization + * ``request_headers`` - the request headers (in raw HTTP format) -The directory name is made from the request fingerprint (see -``scrapy.utils.request.fingerprint``), and one level of subdirectories is -used to avoid creating too many files into the same directory (which is -inefficient in many file systems). An example directory could be:: + * ``response_body`` - the plain response body - /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 + * ``response_headers`` - the request headers (in raw HTTP format) + + * ``meta`` - some metadata of this cache resource in Python ``repr()`` + format (grep-friendly format) + + * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more + efficient deserialization + + The directory name is made from the request fingerprint (see + ``scrapy.utils.request.fingerprint``), and one level of subdirectories is + used to avoid creating too many files into the same directory (which is + inefficient in many file systems). An example directory could be:: + + /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 .. _httpcache-storage-dbm: DBM storage backend ~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.13 +.. class:: DbmCacheStorage -A DBM_ storage backend is also available for the HTTP cache middleware. + .. versionadded:: 0.13 -By default, it uses the anydbm_ module, but you can change it with the -:setting:`HTTPCACHE_DBM_MODULE` setting. + A DBM_ storage backend is also available for the HTTP cache middleware. -In order to use this storage backend, set: + By default, it uses the :mod:`dbm`, but you can change it with the + :setting:`HTTPCACHE_DBM_MODULE` setting. -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage`` +.. _httpcache-storage-custom: -.. _httpcache-storage-leveldb: +Writing your own storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -LevelDB storage backend -~~~~~~~~~~~~~~~~~~~~~~~ +You can implement a cache storage backend by creating a Python class that +defines the methods described below. -.. versionadded:: 0.23 +.. module:: scrapy.extensions.httpcache -A LevelDB_ storage backend is also available for the HTTP cache middleware. +.. class:: CacheStorage -This backend is not recommended for development because only one process can -access LevelDB databases at the same time, so you can't run a crawl and open -the scrapy shell in parallel for the same spider. + .. method:: open_spider(spider) -In order to use this storage backend: + This method gets called after a spider has been opened for crawling. It handles + the :signal:`open_spider ` signal. -* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage`` -* install `LevelDB python bindings`_ like ``pip install leveldb`` + :param spider: the spider which has been opened + :type spider: :class:`~scrapy.spiders.Spider` object -.. _LevelDB: https://github.com/google/leveldb -.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb + .. method:: close_spider(spider) + + This method gets called after a spider has been closed. It handles + the :signal:`close_spider ` signal. + + :param spider: the spider which has been closed + :type spider: :class:`~scrapy.spiders.Spider` object + + .. method:: retrieve_response(spider, request) + + Return response if present in cache, or ``None`` otherwise. + + :param spider: the spider which generated the request + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the request to find cached response for + :type request: :class:`~scrapy.http.Request` object + + .. method:: store_response(spider, request, response) + + Store the given response in the cache. + + :param spider: the spider for which the response is intended + :type spider: :class:`~scrapy.spiders.Spider` object + + :param request: the corresponding request the spider generated + :type request: :class:`~scrapy.http.Request` object + + :param response: the response to store in the cache + :type response: :class:`~scrapy.http.Response` object + +In order to use your storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class. HTTPCache middleware settings @@ -583,7 +630,7 @@ HTTPCACHE_DBM_MODULE .. versionadded:: 0.13 -Default: ``'anydbm'`` +Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. @@ -623,13 +670,13 @@ Default: ``False`` If enabled, will cache pages unconditionally. A spider may wish to have all responses available in the cache, for -future use with `Cache-Control: max-stale`, for instance. The +future use with ``Cache-Control: max-stale``, for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. -This setting still respects `Cache-Control: no-store` directives in responses. -If you don't want that, filter `no-store` out of the Cache-Control headers in -responses you feedto the cache middleware. +This setting still respects ``Cache-Control: no-store`` directives in responses. +If you don't want that, filter ``no-store`` out of the Cache-Control headers in +responses you feed to the cache middleware. .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS @@ -643,7 +690,7 @@ Default: ``[]`` List of Cache-Control directives in responses to be ignored. Sites often set "no-store", "no-cache", "must-revalidate", etc., but get -upset at the traffic a spider can generate if it respects those +upset at the traffic a spider can generate if it actually respects those directives. This allows to selectively ignore Cache-Control directives that are known to be unimportant for the sites being crawled. @@ -662,6 +709,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. + This middleware also supports decoding `brotli-compressed`_ responses, + provided `brotlipy`_ is installed. + +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotlipy: https://pypi.org/project/brotlipy/ + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -675,16 +728,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_ - HttpProxyMiddleware ------------------- @@ -708,7 +751,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -728,6 +773,17 @@ RedirectMiddleware The urls which the request goes through (while being redirected) can be found in the ``redirect_urls`` :attr:`Request.meta ` key. +.. reqmeta:: redirect_reasons + +The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the +``redirect_reasons`` :attr:`Request.meta ` key. For +example: ``[301, 302, 307, 'meta refresh']``. + +The format of a reason depends on the middleware that handled the corresponding +redirect. For example, :class:`RedirectMiddleware` indicates the triggering +response status code as an integer, while :class:`MetaRefreshMiddleware` +always uses the ``'meta refresh'`` string as reason. + The :class:`RedirectMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -776,7 +832,7 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be follow for a single request. +The maximum number of redirections that will be followed for a single request. MetaRefreshMiddleware --------------------- @@ -789,10 +845,12 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` -This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect` -and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware` +This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, +:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described +for :class:`RedirectMiddleware` MetaRefreshMiddleware settings @@ -809,6 +867,19 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``[]`` + +Meta tags within these tags are ignored. + +.. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY @@ -833,8 +904,6 @@ RetryMiddleware Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. -Once there are no more failed pages to retry, this middleware sends a signal -(retry_complete), so other extensions could connect to that signal. The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -871,12 +940,17 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. +Maximum number of retries can also be specified per-request using +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +When initialized, the :reqmeta:`max_retry_times` meta key takes higher +precedence over the :setting:`RETRY_TIMES` setting. + .. setting:: RETRY_HTTP_CODES RETRY_HTTP_CODES ^^^^^^^^^^^^^^^^ -Default: ``[500, 502, 503, 504, 408]`` +Default: ``[500, 502, 503, 504, 522, 524, 408, 429]`` Which HTTP response codes to retry. Other errors (DNS lookup issues, connections lost, etc) are always retried. @@ -902,6 +976,24 @@ RobotsTxtMiddleware To make sure Scrapy respects robots.txt make sure the middleware is enabled and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the + user agent string to use for matching in the robots.txt_ file. If it + is ``None``, the User-Agent header you are sending with the request or the + :setting:`USER_AGENT` setting (in that order) will be used for determining + the user agent to use in the robots.txt_ file. + + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` + * :ref:`Reppy ` + * :ref:`Robotexclusionrulesparser ` + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser `. + .. reqmeta:: dont_obey_robotstxt If :attr:`Request.meta ` has @@ -909,6 +1001,129 @@ If :attr:`Request.meta ` has the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. +Parsers vary in several aspects: + +* Language of implementation + +* Supported specification + +* Support for wildcard matching + +* Usage of `length based rule `_: + in particular for ``Allow`` and ``Disallow`` directives, where the most + specific rule based on the length of the path trumps the less specific + (shorter) rule + +Performance comparison of different parsers is available at `the following link +`_. + +.. _protego-parser: + +Protego parser +~~~~~~~~~~~~~~ + +Based on `Protego `_: + +* implemented in Python + +* is compliant with `Google's Robots.txt Specification + `_ + +* supports wildcard matching + +* uses the length based rule + +Scrapy uses this parser by default. + +.. _python-robotfileparser: + +RobotFileParser +~~~~~~~~~~~~~~~ + +Based on `RobotFileParser +`_: + +* is Python's built-in robots.txt_ parser + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* lacks support for wildcard matching + +* doesn't use the length based rule + +It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. + +In order to use this parser, set: + +* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` + +.. _reppy-parser: + +Reppy parser +~~~~~~~~~~~~ + +Based on `Reppy `_: + +* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ + `_ + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* supports wildcard matching + +* uses the length based rule + +Native implementation, provides better speed than Protego. + +In order to use this parser: + +* Install `Reppy `_ by running ``pip install reppy`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.ReppyRobotParser`` + +.. _rerp-parser: + +Robotexclusionrulesparser +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Based on `Robotexclusionrulesparser `_: + +* implemented in Python + +* is compliant with `Martijn Koster's 1996 draft specification + `_ + +* supports wildcard matching + +* doesn't use the length based rule + +In order to use this parser: + +* Install `Robotexclusionrulesparser `_ by running + ``pip install robotexclusionrulesparser`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.RerpRobotParser`` + +.. _support-for-new-robots-parser: + +Implementing support for a new parser +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can implement support for a new robots.txt_ parser by subclassing +the abstract base class :class:`~scrapy.robotstxt.RobotParser` and +implementing the methods described below. + +.. module:: scrapy.robotstxt + :synopsis: robots.txt parser interface and implementations + +.. autoclass:: RobotParser + :members: + +.. _robots.txt: https://www.robotstxt.org/ DownloaderStats --------------- @@ -924,6 +1139,16 @@ DownloaderStats To use this middleware you must enable the :setting:`DOWNLOADER_STATS` setting. + +UriUserinfoMiddleware +--------------------- + +.. module:: scrapy.downloadermiddlewares.uriuserinfo + :synopsis: URI Userinfo Middleware + +.. autoclass:: UriUserinfoMiddleware + + UserAgentMiddleware ------------------- @@ -934,7 +1159,7 @@ UserAgentMiddleware Middleware that allows spiders to override the default user agent. - In order for a spider to override the default user agent, its `user_agent` + In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. .. _ajaxcrawl-middleware: @@ -948,7 +1173,7 @@ AjaxCrawlMiddleware Middleware that finds 'AJAX crawlable' page variants based on meta-fragment html tag. See - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started + https://developers.google.com/search/docs/ajax-crawling/docs/getting-started for more info. .. note:: @@ -976,8 +1201,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ @@ -987,5 +1220,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f6493d155..7d3026455 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -30,6 +30,8 @@ Python `import search path`_. .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path +.. _populating-settings: + Populating the settings ======================= @@ -122,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in extensions, middlewares and item pipelines:: - class MyExtension(object): + class MyExtension: def __init__(self, log_is_enabled=False): if log_is_enabled: print("log is enabled!") @@ -178,6 +180,44 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_ENDPOINT_URL + +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like storage, for example Minio or s3.scality. + +.. setting:: AWS_USE_SSL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with +S3 or S3-like storage. By default SSL will be used. + +.. setting:: AWS_VERIFY + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default +SSL verification will occur. + +.. setting:: AWS_REGION_NAME + +AWS_REGION_NAME +--------------- + +Default: ``None`` + +The name of the region associated with the AWS client. + .. setting:: BOT_NAME BOT_NAME @@ -186,8 +226,7 @@ BOT_NAME Default: ``'scrapybot'`` The name of the bot implemented by this Scrapy project (also known as the -project name). This will be used to construct the User-Agent by default, and -also for logging. +project name). This name will be used for the logging too. It's automatically populated with your project name when you create your project with the :command:`startproject` command. @@ -209,7 +248,7 @@ CONCURRENT_REQUESTS Default: ``16`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed by the Scrapy downloader. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -219,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN Default: ``8`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. See also: :ref:`topics-autothrottle` and its @@ -233,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP Default: ``0`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single IP. If non-zero, the :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is used instead. In other words, concurrency limits will be applied per IP, not @@ -290,16 +329,16 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the request priority based on its depth: +An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of +a :class:`~scrapy.http.Request` based on its depth. -- if zero (default), no priority adjustment is made from depth -- **a positive value will decrease the priority, i.e. higher depth - requests will be processed later** ; this is commonly used when doing - breadth-first crawls (BFO) -- a negative value will increase priority, i.e., higher depth requests - will be processed sooner (DFO) +The priority of a request is adjusted as follows:: -See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. + request.priority = request.priority - ( depth * DEPTH_PRIORITY ) + +As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request +priority (BFO), while negative values increase request priority (DFO). See +also :ref:`faq-bfo-dfo`. .. note:: @@ -307,17 +346,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO. other priority settings :setting:`REDIRECT_PRIORITY_ADJUST` and :setting:`RETRY_PRIORITY_ADJUST`. -.. setting:: DEPTH_STATS - -DEPTH_STATS ------------ - -Default: ``True`` - -Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` - -Whether to collect maximum depth stats. - .. setting:: DEPTH_STATS_VERBOSE DEPTH_STATS_VERBOSE @@ -348,6 +376,21 @@ Default: ``10000`` DNS in-memory cache size. +.. setting:: DNS_RESOLVER + +DNS_RESOLVER +------------ + +.. versionadded:: 2.0 + +Default: ``'scrapy.resolver.CachingThreadedResolver'`` + +The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver`` +supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting, +but works only with IPv4 addresses. Scrapy provides an alternative resolver, +``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not +take the :setting:`DNS_TIMEOUT` setting into account. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -408,9 +451,29 @@ or even enable client-side authentication (and various other things). which uses the platform's certificates to validate remote endpoints. **This is only available if you use Twisted>=14.0.** -If you do use a custom ContextFactory, make sure it accepts a ``method`` -parameter at init (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`). +If you do use a custom ContextFactory, make sure its ``__init__`` method +accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping +:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging`` +parameter (``bool``) and a ``tls_ciphers`` parameter (see +:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`). + +.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS + +DOWNLOADER_CLIENT_TLS_CIPHERS +----------------------------- + +Default: ``'DEFAULT'`` + +Use this setting to customize the TLS/SSL ciphers used by the default +HTTP/1.1 downloader. + +The setting should contain a string in the `OpenSSL cipher list format`_, +these ciphers will be used as client ciphers. Changing this setting may be +necessary to access certain HTTPS websites: for example, you may need to use +``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a +specific cipher that is not included in ``DEFAULT`` if a website requires it. + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT .. setting:: DOWNLOADER_CLIENT_TLS_METHOD @@ -438,6 +501,20 @@ This setting must be one of these string values: We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 or above (Twisted>=14.0 if you can). +.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING + +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING +------------------------------------- + +Default: ``False`` + +Setting this to ``True`` will enable DEBUG level messages about TLS connection +parameters after establishing HTTPS connections. The kind of information logged +depends on the versions of OpenSSL and pyOpenSSL. + +This setting is only used for the default +:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES @@ -457,7 +534,8 @@ Default:: { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, @@ -509,6 +587,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced per ip address instead of per domain. +.. _spider-download_delay-attribute: + You can also change this setting per spider by setting ``download_delay`` spider attribute. @@ -570,7 +650,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024MB) +Default: ``1073741824`` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -591,7 +671,7 @@ If you want to disable it set to 0. DOWNLOAD_WARNSIZE ----------------- -Default: `33554432` (32MB) +Default: ``33554432`` (32MB) The response size (in bytes) that downloader will start to warn. @@ -605,6 +685,32 @@ If you want to disable it set to 0. This feature needs Twisted >= 11.1. +.. setting:: DOWNLOAD_FAIL_ON_DATALOSS + +DOWNLOAD_FAIL_ON_DATALOSS +------------------------- + +Default: ``True`` + +Whether or not to fail on broken responses, that is, declared +``Content-Length`` does not match content sent by the server or chunked +response was not properly finish. If ``True``, these responses raise a +``ResponseFailed([_DataLoss])`` error. If ``False``, these responses +are passed through and the flag ``dataloss`` is added to the response, i.e.: +``'dataloss' in response.flags`` is ``True``. + +Optionally, this can be set per-request basis by using the +:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. + +.. note:: + + A broken response, or data loss error, may happen under several + circumstances, from server misconfiguration to network errors to data + corruption. It is up to the user to decide if it makes sense to process + broken responses considering they may contain partial or incomplete content. + If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -621,6 +727,13 @@ override its ``request_fingerprint`` method. This method should accept scrapy :class:`~scrapy.http.Request` object and return its fingerprint (a string). +You can disable filtering of duplicate requests by setting +:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. +Be very careful about this however, because you can get into crawling loops. +It's usually a better idea to set the ``dont_filter`` parameter to +``True`` on the specific :class:`~scrapy.http.Request` that should not be +filtered. + .. setting:: DUPEFILTER_DEBUG DUPEFILTER_DEBUG @@ -636,11 +749,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r EDITOR ------ -Default: `depends on the environment` +Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows) -The editor to use for editing spiders with the :command:`edit` command. It -defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults -to ``vi`` (on Unix systems) or the IDLE editor (on Windows). +The editor to use for editing spiders with the :command:`edit` command. +Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit` +command will prefer it over the default setting. .. setting:: EXTENSIONS @@ -687,6 +800,57 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FTP_PASSIVE_MODE + +FTP_PASSIVE_MODE +---------------- + +Default: ``True`` + +Whether or not to use passive mode when initiating FTP transfers. + +.. reqmeta:: ftp_password +.. setting:: FTP_PASSWORD + +FTP_PASSWORD +------------ + +Default: ``"guest"`` + +The password to use for FTP connections when there is no ``"ftp_password"`` +in ``Request`` meta. + +It can be overriden in a request in any of the following ways: + +- Specifying ``ftp_password`` in :attr:`Request.meta ` + +- Specifying the password in :attr:`Request.url ` + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) + +.. note:: + Paraphrasing `RFC 1635`_, although it is common to use either the password + "guest" or one's e-mail address for anonymous FTP, + some FTP servers explicitly ask for the user's e-mail address + and will not allow login with the "guest" password. + +.. _RFC 1635: https://tools.ietf.org/html/rfc1635 + +.. reqmeta:: ftp_user +.. setting:: FTP_USER + +FTP_USER +-------- + +Default: ``"anonymous"`` + +The default username to use for FTP connections. + +It can be overriden in a request in any of the following ways: + +- Specifying ``ftp_user`` in :attr:`Request.meta ` + +- Specifying the username in :attr:`Request.url ` + (see :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware`) .. setting:: ITEM_PIPELINES @@ -750,7 +914,7 @@ LOG_FORMAT Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` -String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available +String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available placeholders. .. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes @@ -768,6 +932,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: :class:`scrapy.logformatter.LogFormatter` + +The class to use for :ref:`formatting log messages ` for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL @@ -786,9 +959,29 @@ LOG_STDOUT Default: ``False`` If ``True``, all standard output (and error) of your process will be redirected -to the log. For example if you ``print 'hello'`` it will appear in the Scrapy +to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. +.. setting:: LOG_SHORT_NAMES + +LOG_SHORT_NAMES +--------------- + +Default: ``False`` + +If ``True``, the logs will just contain the root path. If it is set to ``False`` +then it displays the component responsible for the log output + +.. setting:: LOGSTATS_INTERVAL + +LOGSTATS_INTERVAL +----------------- + +Default: ``60.0`` + +The interval (in seconds) between each logging printout of the stats +by :class:`~scrapy.extensions.logstats.LogStats`. + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED @@ -818,13 +1011,15 @@ Example:: MEMUSAGE_ENABLED ---------------- -Default: ``False`` +Default: ``True`` Scope: ``scrapy.extensions.memusage`` -Whether to enable the memory usage extension that will shutdown the Scrapy -process when it exceeds a memory limit, and also notify by email when that -happened. +Whether to enable the memory usage extension. This extension keeps track of +a peak memory used by the process (it writes it to stats). It can also +optionally shutdown the Scrapy process when it exceeds a memory limit +(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened +(see :setting:`MEMUSAGE_NOTIFY_MAIL`). See :ref:`topics-extensions-ref-memusage`. @@ -879,19 +1074,6 @@ Example:: See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_REPORT - -MEMUSAGE_REPORT ---------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -Whether to send a memory usage report after each spider has been closed. - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -935,7 +1117,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option. If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. -.. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. _wget: https://www.gnu.org/software/wget/manual/wget.html .. setting:: REACTOR_THREADPOOL_MAXSIZE @@ -1006,6 +1188,28 @@ If enabled, Scrapy will respect robots.txt policies. For more information see this option is enabled by default in settings.py file generated by ``scrapy startproject`` command. +.. setting:: ROBOTSTXT_PARSER + +ROBOTSTXT_PARSER +---------------- + +Default: ``'scrapy.robotstxt.ProtegoRobotParser'`` + +The parser backend to use for parsing ``robots.txt`` files. For more information see +:ref:`topics-dlmw-robots`. + +.. setting:: ROBOTSTXT_USER_AGENT + +ROBOTSTXT_USER_AGENT +^^^^^^^^^^^^^^^^^^^^ + +Default: ``None`` + +The user agent string to use for matching in the robots.txt file. If ``None``, +the User-Agent header you are sending with the request or the +:setting:`USER_AGENT` setting (in that order) will be used for determining +the user agent to use in the robots.txt file. + .. setting:: SCHEDULER SCHEDULER @@ -1028,11 +1232,59 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + 1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request: - reason: cannot serialize (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) + +.. setting:: SCHEDULER_DISK_QUEUE + +SCHEDULER_DISK_QUEUE +-------------------- + +Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` + +Type of disk queue that will be used by scheduler. Other available types are +``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +``scrapy.squeues.MarshalLifoDiskQueue``. + +.. setting:: SCHEDULER_MEMORY_QUEUE + +SCHEDULER_MEMORY_QUEUE +---------------------- +Default: ``'scrapy.squeues.LifoMemoryQueue'`` + +Type of in-memory queue used by scheduler. Other available type is: +``scrapy.squeues.FifoMemoryQueue``. + +.. setting:: SCHEDULER_PRIORITY_QUEUE + +SCHEDULER_PRIORITY_QUEUE +------------------------ +Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` + +Type of priority queue used by the scheduler. Another available type is +``scrapy.pqueues.DownloaderAwarePriorityQueue``. +``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than +``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different +domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. + +.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE + +SCRAPER_SLOT_MAX_ACTIVE_SIZE +---------------------------- + +.. versionadded:: 2.0 + +Default: ``5_000_000`` + +Soft limit (in bytes) for response data being processed. + +While the sum of the sizes of all responses being processed is above this value, +Scrapy does not process new requests. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS @@ -1056,7 +1308,7 @@ Default:: 'scrapy.contracts.default.ScrapesContract': 3, } -A dict containing the scrapy contracts enabled by default in Scrapy. You should +A dict containing the Scrapy contracts enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` instead. For more info see :ref:`topics-contracts`. @@ -1078,6 +1330,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. setting:: SPIDER_LOADER_WARN_ONLY + +SPIDER_LOADER_WARN_ONLY +----------------------- + +.. versionadded:: 1.3.3 + +Default: ``False`` + +By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +it will fail loudly if there is any ``ImportError`` exception. +But you can choose to silence this exception and turn it into a simple +warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. + +.. note:: + Some :ref:`scrapy commands ` run with this setting to ``True`` + already (i.e. they will only issue a warning and will not fail) + since they do not actually need to load spider classes to work: + :command:`scrapy runspider `, + :command:`scrapy settings `, + :command:`scrapy startproject `, + :command:`scrapy version `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES @@ -1187,6 +1462,101 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR + +TWISTED_REACTOR +--------------- + +.. versionadded:: 2.0 + +Default: ``None`` + +Import path of a given :mod:`~twisted.internet.reactor`. + +Scrapy will install this reactor if no other reactor is installed yet, such as +when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.CrawlerProcess` class. + +If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +need to install the correct reactor manually. You can do that using +:func:`~scrapy.utils.reactor.install_reactor`: + +.. autofunction:: scrapy.utils.reactor.install_reactor + +If a reactor is already installed, +:func:`~scrapy.utils.reactor.install_reactor` has no effect. + +:meth:`CrawlerRunner.__init__ ` raises +:exc:`Exception` if the installed reactor does not match the +:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:mod:`~twisted.internet.reactor` imports in project files and imported +third-party libraries will make Scrapy raise :exc:`Exception` when +it checks which reactor is installed. + +In order to use the reactor installed by Scrapy:: + + import scrapy + from twisted.internet import reactor + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + + +which raises :exc:`Exception`, becomes:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + from twisted.internet import reactor + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + + +The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which +means that Scrapy will not attempt to install any specific reactor, and the +default reactor defined by Twisted for the current platform will be used. This +is to maintain backward compatibility and avoid possible problems caused by +using a non-default reactor. + +For additional information, see :doc:`core/howto/choosing-reactor`. + .. setting:: URLLENGTH_LIMIT @@ -1198,16 +1568,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html .. setting:: USER_AGENT USER_AGENT ---------- -Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` -The default User-Agent to use when crawling, unless overridden. +The default User-Agent to use when crawling, unless overridden. This user agent is +also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` +if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and +there is no overridding User-Agent header specified for the request. Settings documented elsewhere: diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py index 19fc303b5..a37ffa0dc 100644 --- a/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -1,10 +1,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.auth` instead", + "use `scrapy.downloadermiddlewares.httpauth` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.utils.deprecate import create_deprecated_class -from scrapy.downloadermiddlewares.auth import AuthMiddleware - -HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) +from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py deleted file mode 100644 index 64239ab45..000000000 --- a/scrapy/downloadermiddlewares/auth.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -HTTP/FTP Authorization downloader middleware - -See documentation in docs/topics/downloader-middleware.rst -""" -from six.moves.urllib.parse import unquote, urlunparse - -from w3lib.http import basic_auth_header - -from scrapy import signals -from scrapy.utils.httpobj import urlparse_cached - - -def credstrip_url(parsed_url): - """Strip username and password from an urlparse'd URL""" - return urlunparse(( - parsed_url.scheme, - parsed_url.netloc.split('@')[-1], - parsed_url.path, - parsed_url.params, - parsed_url.query, - parsed_url.fragment)) - - -def _unquote(s): - if s is not None: - return unquote(s) - - -class AuthMiddleware(object): - """ - Populate authorization credentials for HTTP and FTP requests. - - For http(s):// requests, set Basic HTTP Authorization header, - either from http_user and http_pass spider attributes, - or from URL netloc parsing. - - Also handle FTP credentials from ftp://user:password@... URLs, - populating request's meta accordingly. - """ - - @classmethod - def from_crawler(cls, crawler): - o = cls() - crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) - return o - - def spider_opened(self, spider): - usr = getattr(spider, 'http_user', '') - pwd = getattr(spider, 'http_pass', '') - if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - - def process_request(self, request, spider): - url = urlparse_cached(request) - if url.scheme.startswith('http'): - # do not override Auth header set priorly - if 'Authorization' in request.headers: - return - - new_url = None - # credentials from URL override spider attributes - if url.username or url.password: - auth = basic_auth_header(_unquote(url.username), - _unquote(url.password)) - - # no credentials in new url - new_url = credstrip_url(url) - - else: - auth = getattr(self, 'auth', None) - - if auth: - request.headers['Authorization'] = auth - if new_url: - return request.replace(url=new_url) - - elif url.scheme.startswith('ftp'): - if url.username or url.password: - # priorly set credentials take precedence - request.meta.setdefault('ftp_user', _unquote(url.username)) - request.meta.setdefault('ftp_password', _unquote(url.password)) - - # no credentials in new url - return request.replace(url=credstrip_url(url)) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 12b4372e5..7aa7a62bc 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,10 +1,31 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, " - "use `scrapy.downloadermiddlewares.auth` instead", - ScrapyDeprecationWarning) +""" +HTTP basic auth downloader middleware -from scrapy.utils.deprecate import create_deprecated_class -from scrapy.downloadermiddlewares.auth import AuthMiddleware +See documentation in docs/topics/downloader-middleware.rst +""" -HttpAuthMiddleware = create_deprecated_class('HttpAuthMiddleware', AuthMiddleware) +from w3lib.http import basic_auth_header + +from scrapy import signals + + +class HttpAuthMiddleware(object): + """Set Basic HTTP Authorization header + (http_user and http_pass spider class attributes)""" + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o + + def spider_opened(self, spider): + usr = getattr(spider, 'http_user', '') + pwd = getattr(spider, 'http_pass', '') + if usr or pwd: + self.auth = basic_auth_header(usr, pwd) + + def process_request(self, request, spider): + auth = getattr(self, 'auth', None) + if auth and b'Authorization' not in request.headers: + request.headers[b'Authorization'] = auth diff --git a/scrapy/downloadermiddlewares/uriuserinfo.py b/scrapy/downloadermiddlewares/uriuserinfo.py new file mode 100644 index 000000000..a7bff6239 --- /dev/null +++ b/scrapy/downloadermiddlewares/uriuserinfo.py @@ -0,0 +1,49 @@ +from urllib.parse import unquote, urlunparse + +from scrapy.utils.httpobj import urlparse_cached + + +class UriUserinfoMiddleware(object): + """Downloader middleware that replaces `URI userinfo`_ data (user credentials + for HTTP or FTP specified in the request URL) with the corresponding meta + keys for later middlewares or download handlers to use them for + authentication. + + It sets: + + - :reqmeta:`ftp_user` and :reqmeta:`ftp_password` for FTP requests + + - :reqmeta:`http_user` and :reqmeta:`http_pass` for HTTP and HTTPS + requests + + .. _URI userinfo: https://tools.ietf.org/html/rfc2396.html#section-3.2.2 + """ + + def process_request(self, request, spider): + url = urlparse_cached(request) + if url.username is None and url.password is None: + return + + if url.scheme.startswith('http'): + username_field, password_field = 'http_user', 'http_pass' + elif url.scheme.startswith('ftp'): + username_field, password_field = 'ftp_user', 'ftp_password' + else: + return + + for key, value in ((username_field, url.username), + (password_field, url.password)): + if value is not None: + request.meta.setdefault(key, unquote(value)) + + userinfoless_url = urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc.split('@')[-1], + parsed_url.path, + parsed_url.params, + parsed_url.query, + parsed_url.fragment, + ) + ) + return request.replace(url=userinfoless_url) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..5ee6280a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,6 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.downloadermiddlewares.uriuserinfo.UriUserinfoMiddleware': 200, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py deleted file mode 100644 index 6f7a583ef..000000000 --- a/tests/test_downloadermiddleware_auth.py +++ /dev/null @@ -1,186 +0,0 @@ -import unittest - -from scrapy.http import Request -from scrapy.downloadermiddlewares.auth import AuthMiddleware -from scrapy.spiders import Spider - - -class TestSpider(Spider): - http_user = 'foo' - http_pass = 'bar' - - -class NoAuthTestSpider(Spider): - """A test spider that does not set http auth atttributes""" - - -class AuthMiddlewareNoAuthTest(unittest.TestCase): - - def setUp(self): - self.mw = AuthMiddleware() - self.spider = NoAuthTestSpider('bar') - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - def test_no_auth_http(self): - req = Request('http://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn('Authorization', req.headers) - - def test_no_auth_ftp(self): - req = Request('ftp://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertNotIn('ftp_user', req.meta) - self.assertNotIn('ftp_password', req.meta) - - -class AuthMiddlewareTest(unittest.TestCase): - - def setUp(self): - self.mw = AuthMiddleware() - self.spider = TestSpider('foo') - self.mw.spider_opened(self.spider) - - def tearDown(self): - del self.mw - - -class AuthMiddlewareHttpAuthTest(AuthMiddlewareTest): - - def test_auth(self): - req = Request('http://scrapytest.org/') - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') - - def test_auth_already_set(self): - req = Request('http://scrapytest.org/', - headers=dict(Authorization='Digest 123')) - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Digest 123') - - def test_auth_from_http_url(self): - req = Request('http://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_https_url(self): - req = Request('https://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'https://scrapytest.org/') - - def test_auth_from_http_url_no_spider_attrs(self): - class AnotherTestSpider(Spider): - pass - req = Request('http://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, AnotherTestSpider('bar')) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_https_url_no_spider_attrs(self): - class AnotherTestSpider(Spider): - pass - req = Request('https://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, AnotherTestSpider('bar')) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') - self.assertEquals(new_req.url, 'https://scrapytest.org/') - - def test_auth_from_http_url_empty_pass(self): - req = Request('http://username:@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_http_url_pass_none(self): - req = Request('http://username@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6Tm9uZQ==') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - def test_auth_from_http_url_empty_user(self): - req = Request('http://:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.headers['Authorization'], b'Basic OnBhc3N3b3Jk') - self.assertEquals(new_req.url, 'http://scrapytest.org/') - - -class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): - - def test_no_auth_from_ftp_url_meta_unchanged(self): - usr, pwd = 'u', 'p' - req = Request('ftp://scrapytest.org/', - meta={"ftp_user": usr, "ftp_password": pwd}) - assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.meta['ftp_user'], usr) - self.assertEquals(req.meta['ftp_password'], pwd) - - def test_auth_from_ftp_url_meta_unchanged(self): - """Request's meta credentials are kept as-is, - but URL is stripped from credentials - """ - usr, pwd = 'u', 'p' - req = Request('ftp://username:password@scrapytest.org/', - meta={"ftp_user": usr, "ftp_password": pwd}) - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(new_req.meta['ftp_user'], usr) - self.assertEquals(new_req.meta['ftp_password'], pwd) - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url(self): - req = Request('ftp://username:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_encoded_delims_user(self): - req = Request('ftp://username%3A:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username:') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_encoded_delims_password(self): - req = Request('ftp://username:pass%40word@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], 'pass@word') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_empty_user(self): - req = Request('ftp://:password@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], '') - self.assertEquals(req.meta['ftp_password'], 'password') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_empty_pass(self): - req = Request('ftp://username:@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], '') - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') - - def test_auth_from_ftp_url_pass_none(self): - req = Request('ftp://username@scrapytest.org/') - new_req = self.mw.process_request(req, self.spider) - assert new_req is not None - self.assertEquals(req.meta['ftp_user'], 'username') - self.assertEquals(req.meta['ftp_password'], None) - self.assertEquals(new_req.url, 'ftp://scrapytest.org/') diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py new file mode 100644 index 000000000..425a5cc79 --- /dev/null +++ b/tests/test_downloadermiddleware_httpauth.py @@ -0,0 +1,32 @@ +import unittest + +from scrapy.http import Request +from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.spiders import Spider + + +class TestSpider(Spider): + http_user = 'foo' + http_pass = 'bar' + + +class HttpAuthMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpider('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + + def test_auth_already_set(self): + req = Request('http://scrapytest.org/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_uriuserinfo.py b/tests/test_downloadermiddleware_uriuserinfo.py new file mode 100644 index 000000000..13dcd4736 --- /dev/null +++ b/tests/test_downloadermiddleware_uriuserinfo.py @@ -0,0 +1,72 @@ +import unittest + +from scrapy.http import Request +from scrapy.downloadermiddlewares.uriuserinfo import UriUserinfoMiddleware +from scrapy.spiders import Spider + + +class BaseTestCase: + + class Implementation(unittest.TestCase): + + def setUp(self): + self.mw = UriUserinfoMiddleware() + self.spider = Spider('bar') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_username_and_password(self): + req = Request('{}://foo:bar@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertEqual(req.meta[self.password_field], 'bar') + + def test_username_and_empty_password(self): + req = Request('{}://foo:@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertEqual(req.meta[self.password_field], '') + + def test_username_and_no_password(self): + req = Request('{}://foo@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo') + self.assertNotIn(self.password_field, req.meta) + + def test_empty_username_and_nonempty_password(self): + req = Request('{}://:bar@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], '') + self.assertEqual(req.meta[self.password_field], 'bar') + + def test_no_username_and_no_password(self): + req = Request('{}://scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn(self.username_field, req.meta) + self.assertNotIn(self.password_field, req.meta) + + def test_unquoting(self): + req = Request('{}://foo%3A:b%40r@scrapytest.org/'.format(self.protocol)) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.meta[self.username_field], 'foo:') + self.assertEqual(req.meta[self.password_field], 'b@r') + + +class UriUserinfoMiddlewareFTPTest(BaseTestCase.Implementation): + protocol = 'ftp' + username_field = 'ftp_user' + password_field = 'ftp_password' + + +class UriUserinfoMiddlewareHTTPTest(BaseTestCase.Implementation): + protocol = 'http' + username_field = 'http_user' + password_field = 'http_pass' + + +class UriUserinfoMiddlewareHTTPSTest(BaseTestCase.Implementation): + protocol = 'https' + username_field = 'http_user' + password_field = 'http_pass'