diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f347a0cd0..de22a2783 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0 +current_version = 2.1.0 commit = True tag = True tag_name = {new_version} diff --git a/.travis.yml b/.travis.yml index 66e1a9617..6bde973f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,25 +11,23 @@ matrix: python: 3.8 - env: TOXENV=flake8 python: 3.8 + - env: TOXENV=docs + python: 3.7 # Keep in sync with .readthedocs.yml + - env: TOXENV=pypy3 - - env: TOXENV=py35 - python: 3.5 + - python: 3.5 - env: TOXENV=pinned python: 3.5 - - env: TOXENV=py35-asyncio + - env: TOXENV=asyncio python: 3.5.2 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 + - python: 3.6 + - python: 3.7 + - env: PYPI_RELEASE_JOB=true python: 3.8 - env: TOXENV=extra-deps python: 3.8 - - env: TOXENV=py38-asyncio + - env: TOXENV=asyncio python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml install: - | if [ "$TOXENV" = "pypy3" ]; then @@ -62,4 +60,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" + condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/docs/news.rst b/docs/news.rst index e9b7140cd..a158246eb 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,153 @@ Release notes ============= +.. _release-2.1.0: + +Scrapy 2.1.0 (2020-04-24) +------------------------- + +Highlights: + +* New :setting:`FEEDS` setting to export to multiple feeds +* New :attr:`Response.ip_address ` attribute + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :exc:`AssertionError` exceptions triggered by :ref:`assert ` + statements have been replaced by new exception types, to support running + Python in optimized mode (see :option:`-O`) without changing Scrapy’s + behavior in any unexpected ways. + + If you catch an :exc:`AssertionError` exception from Scrapy, update your + code to catch the corresponding new exception. + + (:issue:`4440`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use + :setting:`SCHEDULER_DEBUG` instead (:issue:`4385`) + +* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use + :setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`) + +* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` + middleware has been removed, including the entire + :class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers + work out of the box (:issue:`4431`) + +* The ``spiders`` property has been removed from + :class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader + ` or instantiate + :setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`) + +* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode`` + classes have been removed from :mod:`scrapy.utils.datatypes` + (:issue:`4400`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in + favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`, + :issue:`4507`) + + +New features +~~~~~~~~~~~~ + +* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds + with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support multiple + ``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support + specifying an output format by appending ``:`` to the output file + (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The new :attr:`Response.ip_address ` + attribute gives access to the IP address that originated a response + (:issue:`3903`, :issue:`3940`) + +* A warning is now issued when a value in + :attr:`~scrapy.spiders.Spider.allowed_domains` includes a port + (:issue:`50`, :issue:`3198`, :issue:`4413`) + +* Zsh completion now excludes used option aliases from the completion list + (:issue:`4438`) + + +Bug fixes +~~~~~~~~~ + +* :ref:`Request serialization ` no longer breaks for + callbacks that are spider attributes which are assigned a function with a + different name (:issue:`4500`) + +* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer + cause a :exc:`TypeError` exception (:issue:`4410`) + +* Zsh completion no longer allows options after arguments (:issue:`4438`) + +* zope.interface 5.0.0 and later versions are now supported + (:issue:`4447`, :issue:`4448`) + +* :meth:`Spider.make_requests_from_url + `, deprecated in Scrapy + 1.4.0, now issues a warning when used (:issue:`4412`) + + +Documentation +~~~~~~~~~~~~~ + +* Improved the documentation about signals that allow their handlers to + return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`, + :issue:`4390`) + +* Our PyPI entry now includes links for our documentation, our source code + repository and our issue tracker (:issue:`4456`) + +* Covered the `curl2scrapy `_ + service in the documentation (:issue:`4206`, :issue:`4455`) + +* Removed references to the Guppy library, which only works in Python 2 + (:issue:`4285`, :issue:`4343`) + +* Extended use of InterSphinx to link to Python 3 documentation + (:issue:`4444`, :issue:`4445`) + +* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`, + :issue:`4496`, :issue:`4503`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Removed warnings about using old, removed settings (:issue:`4404`) + +* Removed a warning about importing + :class:`~twisted.internet.testing.StringTransport` from + ``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`) + +* Removed outdated Debian package build files (:issue:`4384`) + +* Removed :class:`object` usage as a base class (:issue:`4430`) + +* Removed code that added support for old versions of Twisted that we no + longer support (:issue:`4472`) + +* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`, + :issue:`4481`) + +* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`, + :issue:`4446`, :issue:`4489`) + + .. _release-2.0.1: Scrapy 2.0.1 (2020-03-18) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 5f61d6796..7a9ecd4d5 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -7,10 +7,6 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related API and behavior changes without a - deprecation period or warning. - .. _coroutine-support: Supported callables diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index e3e8fdc72..cf1de1bd1 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it. ... process = CrawlerProcess(settings={ - 'FEED_FORMAT': 'json', - 'FEED_URI': 'items.json' + "FEEDS": { + "items.json": {"format": "json"}, + }, }) process.crawl(MySpider) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 5eb4915cd..024f46466 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -619,6 +619,9 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + .. versionadded:: 2.1.0 + The ``ip_address`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -710,6 +713,8 @@ Response objects .. attribute:: Response.ip_address + .. versionadded:: 2.1.0 + The IP address of the server from which the Response originated. This attribute is currently only populated by the HTTP 1.1 download diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 18f81838f..f06d9db3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``). .. note:: HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, - unless you use Twisted<11.1, or if you really want to use HTTP/1.0 - and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme - accordingly, i.e. to - ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. + unless you really want to use HTTP/1.0 and override + :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, + i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY @@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things). Scrapy also has another context factory class that you can set, ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, which uses the platform's certificates to validate remote endpoints. - **This is only available if you use Twisted>=14.0.** If you do use a custom ContextFactory, make sure its ``__init__`` method accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping @@ -494,10 +492,6 @@ This setting must be one of these string values: - ``'TLSv1.2'``: forces TLS version 1.2 - ``'SSLv3'``: forces SSL version 3 (**not recommended**) -.. note:: - - We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 - or above (Twisted>=14.0 if you can). .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -660,8 +654,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_maxsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_WARNSIZE DOWNLOAD_WARNSIZE @@ -679,8 +671,6 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_warnsize` Request.meta key. - This feature needs Twisted >= 11.1. - .. setting:: DOWNLOAD_FAIL_ON_DATALOSS DOWNLOAD_FAIL_ON_DATALOSS diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8f7518b19..0f46f1c87 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -156,6 +156,17 @@ First, we launch the shell:: scrapy shell 'https://scrapy.org' --nolog +.. note:: + + Remember to always enclose URLs in quotes when running the Scrapy shell from + the command line, otherwise URLs containing arguments (i.e. the ``&`` character) + will not work. + + On Windows, use double quotes instead:: + + scrapy shell "https://scrapy.org" --nolog + + Then, the shell fetches the URL (using the Scrapy downloader) and prints the list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 2def53848..8661f86a0 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,8 +16,7 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. -Here is a simple example showing how you can catch signals and perform some action: -:: +Here is a simple example showing how you can catch signals and perform some action:: from scrapy import signals from scrapy import Spider @@ -52,9 +51,45 @@ Deferred signal handlers ======================== Some signals support returning :class:`~twisted.internet.defer.Deferred` -objects from their handlers, see the :ref:`topics-signals-ref` below to know -which ones. +objects from their handlers, allowing you to run asynchronous code that +does not block Scrapy. If a signal handler returns a +:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that +:class:`~twisted.internet.defer.Deferred` to fire. +Let's take an example:: + + class SignalSpider(scrapy.Spider): + name = 'signals' + start_urls = ['http://quotes.toscrape.com/page/1/'] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped) + return spider + + def item_scraped(self, item): + # Send the scraped item to the server + d = treq.post( + 'http://example.com/post', + json.dumps(item).encode('ascii'), + headers={b'Content-Type': [b'application/json']} + ) + + # The next item will be scraped only after + # deferred (d) is fired + return d + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), + } + +See the :ref:`topics-signals-ref` below to know which signals support +:class:`~twisted.internet.defer.Deferred`. .. _topics-signals-ref: @@ -66,9 +101,12 @@ Built-in signals reference Here's the list of Scrapy built-in signals and their meaning. -engine_started +Engine signals -------------- +engine_started +~~~~~~~~~~~~~~ + .. signal:: engine_started .. function:: engine_started() @@ -81,7 +119,7 @@ engine_started getting fired before :signal:`spider_opened`. engine_stopped --------------- +~~~~~~~~~~~~~~ .. signal:: engine_stopped .. function:: engine_stopped() @@ -91,9 +129,20 @@ engine_stopped This signal supports returning deferreds from their handlers. -item_scraped +Item signals ------------ +.. note:: + As at max :setting:`CONCURRENT_ITEMS` items are processed in + parallel, many deferreds are fired together using + :class:`~twisted.internet.defer.DeferredList`. Hence the next + batch waits for the :class:`~twisted.internet.defer.DeferredList` + to fire and then runs the respective item signal handler for + the next batch of scraped items. + +item_scraped +~~~~~~~~~~~~ + .. signal:: item_scraped .. function:: item_scraped(item, response, spider) @@ -112,7 +161,7 @@ item_scraped :type response: :class:`~scrapy.http.Response` object item_dropped ------------- +~~~~~~~~~~~~ .. signal:: item_dropped .. function:: item_dropped(item, response, exception, spider) @@ -137,7 +186,7 @@ item_dropped :type exception: :exc:`~scrapy.exceptions.DropItem` exception item_error ------------- +~~~~~~~~~~ .. signal:: item_error .. function:: item_error(item, response, spider, failure) @@ -159,8 +208,11 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure +Spider signals +-------------- + spider_closed -------------- +~~~~~~~~~~~~~ .. signal:: spider_closed .. function:: spider_closed(spider, reason) @@ -183,7 +235,7 @@ spider_closed :type reason: str spider_opened -------------- +~~~~~~~~~~~~~ .. signal:: spider_opened .. function:: spider_opened(spider) @@ -198,7 +250,7 @@ spider_opened :type spider: :class:`~scrapy.spiders.Spider` object spider_idle ------------ +~~~~~~~~~~~ .. signal:: spider_idle .. function:: spider_idle(spider) @@ -228,7 +280,7 @@ spider_idle due to duplication). spider_error ------------- +~~~~~~~~~~~~ .. signal:: spider_error .. function:: spider_error(failure, response, spider) @@ -246,8 +298,11 @@ spider_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.spiders.Spider` object +Request signals +--------------- + request_scheduled ------------------ +~~~~~~~~~~~~~~~~~ .. signal:: request_scheduled .. function:: request_scheduled(request, spider) @@ -264,7 +319,7 @@ request_scheduled :type spider: :class:`~scrapy.spiders.Spider` object request_dropped ---------------- +~~~~~~~~~~~~~~~ .. signal:: request_dropped .. function:: request_dropped(request, spider) @@ -281,7 +336,7 @@ request_dropped :type spider: :class:`~scrapy.spiders.Spider` object request_reached_downloader ---------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~ .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) @@ -297,7 +352,7 @@ request_reached_downloader :type spider: :class:`~scrapy.spiders.Spider` object request_left_downloader ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) @@ -315,8 +370,11 @@ request_left_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +Response signals +---------------- + response_received ------------------ +~~~~~~~~~~~~~~~~~ .. signal:: response_received .. function:: response_received(response, request, spider) @@ -336,7 +394,7 @@ response_received :type spider: :class:`~scrapy.spiders.Spider` object response_downloaded -------------------- +~~~~~~~~~~~~~~~~~~~ .. signal:: response_downloaded .. function:: response_downloaded(response, request, spider) diff --git a/pytest.ini b/pytest.ini index ff923a2de..f26e274d7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,64 +30,56 @@ flake8-ignore = # Issues pending a review: # scrapy/commands scrapy/commands/__init__.py E128 - scrapy/commands/fetch.py E401 E128 + scrapy/commands/fetch.py E128 scrapy/commands/genspider.py E128 scrapy/commands/parse.py E128 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 - scrapy/commands/startproject.py E127 E128 + scrapy/commands/startproject.py E128 scrapy/commands/version.py E128 # scrapy/contracts - scrapy/contracts/__init__.py W504 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E128 E127 - scrapy/core/scraper.py E128 W504 + scrapy/core/engine.py E128 + scrapy/core/scraper.py E128 scrapy/core/spidermw.py E126 scrapy/core/downloader/contextfactory.py E128 E126 scrapy/core/downloader/webclient.py E128 E126 - scrapy/core/downloader/handlers/ftp.py E128 E127 + scrapy/core/downloader/handlers/ftp.py E128 scrapy/core/downloader/handlers/s3.py E128 E126 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/httpcache.py E126 scrapy/downloadermiddlewares/httpcompression.py E128 - scrapy/downloadermiddlewares/redirect.py W504 scrapy/downloadermiddlewares/retry.py E126 # scrapy/extensions - scrapy/extensions/closespider.py E128 E123 + scrapy/extensions/closespider.py E128 scrapy/extensions/feedexport.py E128 scrapy/extensions/httpcache.py E128 - scrapy/extensions/telnet.py W504 # scrapy/http - scrapy/http/request/form.py E123 scrapy/http/response/__init__.py E128 scrapy/http/response/text.py E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E402 W504 + scrapy/linkextractors/__init__.py E402 # scrapy/loader scrapy/loader/__init__.py E128 # scrapy/pipelines scrapy/pipelines/files.py E116 - scrapy/pipelines/media.py E125 # scrapy/selector scrapy/selector/__init__.py F403 scrapy/selector/unified.py E111 # scrapy/settings scrapy/settings/default_settings.py E114 E116 # scrapy/spidermiddlewares - scrapy/spidermiddlewares/referer.py E129 W504 + scrapy/spidermiddlewares/referer.py E129 # scrapy/spiders scrapy/spiders/__init__.py E402 # scrapy/utils scrapy/utils/conf.py E402 scrapy/utils/defer.py E128 - scrapy/utils/deprecate.py E128 E127 - scrapy/utils/gz.py W504 scrapy/utils/http.py F403 scrapy/utils/log.py E128 scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 - scrapy/utils/request.py E127 scrapy/utils/response.py E128 scrapy/utils/signal.py E128 scrapy/utils/url.py F403 E128 F405 @@ -101,10 +93,8 @@ flake8-ignore = scrapy/squeues.py E128 # tests tests/__init__.py E402 - tests/mockserver.py E401 E126 E123 + tests/mockserver.py E126 tests/pipelines.py F841 - tests/spiders.py E127 - tests/test_closespider.py E127 tests/test_command_parse.py E128 tests/test_command_shell.py E128 tests/test_commands.py E128 @@ -112,19 +102,18 @@ flake8-ignore = tests/test_crawl.py E741 tests/test_crawler.py F841 tests/test_dependencies.py F841 - tests/test_downloader_handlers.py E124 E127 E128 E126 E123 + tests/test_downloader_handlers.py E124 E128 E126 tests/test_downloadermiddleware_cookies.py E741 E128 E126 - tests/test_downloadermiddleware_decompression.py E127 - tests/test_downloadermiddleware_httpcompression.py E126 E123 + tests/test_downloadermiddleware_httpcompression.py E126 tests/test_downloadermiddleware_httpproxy.py E128 - tests/test_downloadermiddleware_redirect.py E128 E127 + tests/test_downloadermiddleware_redirect.py E128 tests/test_downloadermiddleware_retry.py E128 E126 tests/test_dupefilters.py E741 E128 E124 - tests/test_engine.py E401 E128 + tests/test_engine.py E128 tests/test_exporters.py E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py F841 - tests/test_http_request.py E402 E127 E128 E128 E126 E123 + tests/test_http_request.py E402 E128 E128 E126 tests/test_http_response.py E128 tests/test_item.py E128 F841 tests/test_linkextractors.py E128 E124 @@ -136,24 +125,23 @@ flake8-ignore = tests/test_pipeline_images.py F841 tests/test_pipeline_media.py E741 E128 tests/test_proxy_connect.py E741 - tests/test_scheduler.py E126 E123 - tests/test_selector.py E127 - tests/test_spidermiddleware_httperror.py E128 E127 E121 + tests/test_scheduler.py E126 + tests/test_spidermiddleware_httperror.py E128 E121 tests/test_spidermiddleware_offsite.py E128 E111 - tests/test_spidermiddleware_referer.py F841 E125 E124 E121 + tests/test_spidermiddleware_referer.py F841 E124 E121 tests/test_squeues.py E741 tests/test_utils_conf.py E128 tests/test_utils_datatypes.py E402 tests/test_utils_defer.py F841 tests/test_utils_deprecate.py F841 - tests/test_utils_http.py E128 W504 + tests/test_utils_http.py E128 tests/test_utils_iterators.py E128 E129 tests/test_utils_log.py E741 tests/test_utils_reqser.py E128 tests/test_utils_request.py E128 tests/test_utils_signal.py E741 F841 tests/test_utils_sitemap.py E128 E124 - tests/test_utils_url.py E127 E125 E126 E123 - tests/test_webclient.py E128 E122 E402 E123 E126 + tests/test_utils_url.py E126 + tests/test_webclient.py E128 E122 E402 E126 tests/test_settings/__init__.py E128 tests/test_spiderloader/__init__.py E128 diff --git a/scrapy/VERSION b/scrapy/VERSION index 227cea215..7ec1d6db4 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index a573a03d9..9f8e6986a 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,7 +23,8 @@ class ScrapyCommand: self.settings = None # set in scrapy.cmdline def set_crawler(self, crawler): - assert not hasattr(self, '_crawler'), "crawler already set" + if hasattr(self, '_crawler'): + raise RuntimeError("crawler already set") self._crawler = crawler def syntax(self): diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 41d4f25b2..5af3831a2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -17,10 +17,10 @@ class ContractsManager: self.contracts[contract.name] = contract def tested_methods_from_spidercls(self, spidercls): + is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): - if (callable(value) and value.__doc__ and - re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + if callable(value) and value.__doc__ and is_method(value.__doc__): methods.append(key) return methods diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 3002fc702..a1b0f8f22 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -58,7 +58,11 @@ class ReturnsContract(Contract): def __init__(self, *args, **kwargs): super(ReturnsContract, self).__init__(*args, **kwargs) - assert len(self.args) in [1, 2, 3] + if len(self.args) not in [1, 2, 3]: + raise ValueError( + "Incorrect argument quantity: expected 1, 2 or 3, got %i" + % len(self.args) + ) self.obj_name = self.args[0] or None self.obj_type = self.objects[self.obj_name] diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 432cb1831..94b55c347 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -94,11 +94,12 @@ class FTPDownloadHandler: def gotClient(self, client, request, filepath): self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol)\ - .addCallbacks(callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,)) + return client.retrieveFile(filepath, protocol).addCallbacks( + callback=self._build_response, + callbackArgs=(request, protocol), + errback=self._failed, + errbackArgs=(request,), + ) def _build_response(self, result, request, protocol): self.result = result diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 5a03dcdf7..4c2eea522 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -45,8 +45,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_response(response): - assert response is not None, 'Received None in process_response' - if isinstance(response, Request): + if response is None: + raise TypeError("Received None in process_response") + elif isinstance(response, Request): return response for method in self.methods['process_response']: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 66cf9ad9a..324d21716 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -73,7 +73,8 @@ class ExecutionEngine: @defer.inlineCallbacks def start(self): """Start the execution engine""" - assert not self.running, "Engine already running" + if self.running: + raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True @@ -82,7 +83,8 @@ class ExecutionEngine: def stop(self): """Stop the execution engine gracefully""" - assert self.running, "Engine not running" + if not self.running: + raise RuntimeError("Engine not running") self.running = False dfd = self._close_all_spiders() return dfd.addBoth(lambda _: self._finish_stopping_engine()) @@ -165,7 +167,11 @@ class ExecutionEngine: return d def _handle_downloader_output(self, response, request, spider): - assert isinstance(response, (Request, Response, Failure)), response + if not isinstance(response, (Request, Response, Failure)): + raise TypeError( + "Incorrect type: expected Request, Response or Failure, got %s: %r" + % (type(response), response) + ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): self.crawl(response, spider) @@ -205,8 +211,8 @@ class ExecutionEngine: return not bool(self.slot) def crawl(self, request, spider): - assert spider in self.open_spiders, \ - "Spider %r not opened when crawling: %s" % (spider.name, request) + if spider not in self.open_spiders: + raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) self.schedule(request, spider) self.slot.nextcall.schedule() @@ -224,15 +230,18 @@ class ExecutionEngine: def _downloaded(self, response, slot, request, spider): slot.remove_request(request) - return self.download(response, spider) \ - if isinstance(response, Request) else response + return self.download(response, spider) if isinstance(response, Request) else response def _download(self, request, spider): slot = self.slot slot.add_request(request) def _on_success(response): - assert isinstance(response, (Response, Request)) + if not isinstance(response, (Response, Request)): + raise TypeError( + "Incorrect type: expected Response or Request, got %s: %r" + % (type(response), response) + ) if isinstance(response, Response): response.request = request # tie request to response received logkws = self.logformatter.crawled(request, response, spider) @@ -253,8 +262,8 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): - assert self.has_capacity(), "No free spider slot when opening %r" % \ - spider.name + if not self.has_capacity(): + raise RuntimeError("No free spider slot when opening %r" % spider.name) logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3e4826216..edbb4dd66 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -123,7 +123,11 @@ class Scraper: def _scrape(self, response, request, spider): """Handle the downloaded response or failure through the spider callback/errback""" - assert isinstance(response, (Response, Failure)) + if not isinstance(response, (Response, Failure)): + raise TypeError( + "Incorrect type: expected Response or Failure, got %s: %r" + % (type(response), response) + ) dfd = self._scrape2(response, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, response, spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 20990ea41..6f43771e2 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -78,7 +78,8 @@ class Crawler: @defer.inlineCallbacks def crawl(self, *args, **kwargs): - assert not self.crawling, "Crawling already taking place" + if self.crawling: + raise RuntimeError("Crawling already taking place") self.crawling = True try: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 09ee8377e..b32afb8e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + def process_response(self, request, response, spider): - if (request.meta.get('dont_redirect', False) or - response.status in getattr(spider, 'handle_httpstatus_list', []) or - response.status in request.meta.get('handle_httpstatus_list', []) or - request.meta.get('handle_httpstatus_all', False)): + if ( + request.meta.get('dont_redirect', False) + or response.status in getattr(spider, 'handle_httpstatus_list', []) + or response.status in request.meta.get('handle_httpstatus_list', []) + or request.meta.get('handle_httpstatus_all', False) + ): return response allowed_status = (301, 302, 303, 307, 308) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index e3f212bef..812844c0a 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -20,7 +20,7 @@ class CloseSpider: 'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'), 'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'), 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), - } + } if not any(self.close_on.values()): raise NotConfigured diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 04ffd7235..1663604e7 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username.encode('utf8') and - credentials.checkPassword(self.password.encode('utf8'))): + if not ( + credentials.username == self.username.encode('utf8') + and credentials.checkPassword(self.password.encode('utf8')) + ): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0a6637af8..a98ba9960 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -24,7 +24,8 @@ class Request(object_ref): self.method = str(method).upper() self._set_url(url) self._set_body(body) - assert isinstance(priority, int), "Request priority not an integer: %r" % priority + if not isinstance(priority, int): + raise TypeError("Request priority not an integer: %r" % priority) self.priority = priority if callback is not None and not callable(callback): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af02c8484..cd4e3373f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -178,12 +178,11 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - clickables = [ - el for el in form.xpath( - 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', - namespaces={"re": "http://exslt.org/regular-expressions"}) - ] + clickables = list(form.xpath( + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', + namespaces={"re": "http://exslt.org/regular-expressions"} + )) if not clickables: return diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d0b5066b6..ae019c70f 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -61,8 +61,7 @@ class FilteringLinkExtractor: def __new__(cls, *args, **kwargs): from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if (issubclass(cls, FilteringLinkExtractor) and - not issubclass(cls, LxmlLinkExtractor)): + if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index aab645d3d..a9066986b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -106,7 +106,8 @@ class S3FilesStore: else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection - assert uri.startswith('s3://') + if not uri.startswith("s3://"): + raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -229,6 +230,20 @@ class GCSFilesStore: bucket, prefix = uri[5:].split('/', 1) self.bucket = client.bucket(bucket) self.prefix = prefix + permissions = self.bucket.test_iam_permissions( + ['storage.objects.get', 'storage.objects.create'] + ) + if 'storage.objects.get' not in permissions: + logger.warning( + "No 'storage.objects.get' permission for GSC bucket %(bucket)s. " + "Checking if files are up to date will be impossible. Files will be downloaded every time.", + {'bucket': bucket} + ) + if 'storage.objects.create' not in permissions: + logger.error( + "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!", + {'bucket': bucket} + ) def stat_file(self, path, info): def _onsuccess(blob): @@ -266,7 +281,8 @@ class FTPFilesStore: USE_ACTIVE_MODE = None def __init__(self, uri): - assert uri.startswith('ftp://') + if not uri.startswith("ftp://"): + raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) u = urlparse(uri) self.port = u.port self.host = u.hostname diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 8a0636264..aa65f4f0e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -43,8 +43,7 @@ class MediaPipeline: if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, - settings=None): + def _key_for_pipe(self, key, base_class_name=None, settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -55,8 +54,11 @@ class MediaPipeline: """ class_name = self.__class__.__name__ formatted_key = "{}_{}".format(class_name.upper(), key) - if class_name == base_class_name or not base_class_name \ - or (settings and not settings.get(formatted_key)): + if ( + not base_class_name + or class_name == base_class_name + or settings and not settings.get(formatted_key) + ): return key return formatted_key diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 3784de885..434067b00 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): - if ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + if ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) @@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ((self.tls_protected(response_url) and - self.potentially_trustworthy(request_url)) - or not self.tls_protected(response_url)): + elif ( + self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + or not self.tls_protected(response_url) + ): return self.origin_referrer(response_url) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 36001d982..3dbea5fee 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'): stacklevel=3) -def create_deprecated_class(name, new_class, clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from " - "deprecated class {old}, please inherit " - "from {new}.", - instance_warn_message="{cls} is deprecated, " - "instantiate {new} instead."): +def create_deprecated_class( + name, + new_class, + clsdict=None, + warn_category=ScrapyDeprecationWarning, + warn_once=True, + old_class_path=None, + new_class_path=None, + subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message="{cls} is deprecated, instantiate {new} instead." +): """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c291ae237..fbd7bd18f 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -52,8 +52,7 @@ def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') cenc = response.headers.get('Content-Encoding', b'').lower() - return (_is_gzipped(ctype) or - (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') def gzip_magic_number(response): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b71419111..5e15bf0c8 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): expected_types = (Response, str, bytes) - assert isinstance(obj, expected_types), \ - "obj must be %s, not %s" % ( - " or ".join(t.__name__ for t in expected_types), - type(obj).__name__) + if not isinstance(obj, expected_types): + expected_types_str = " or ".join(t.__name__ for t in expected_types) + raise TypeError( + "Object %r must be %s, not %s" + % (obj, expected_types_str, type(obj).__name__) + ) if isinstance(obj, Response): if not unicode: return obj.body diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5308812d6..3c705f69b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor - assert len(portrange) <= 2, "invalid portrange: %s" % portrange + if len(portrange) > 2: + raise ValueError("invalid portrange: %s" % portrange) if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 749bbc387..5ea2aafb8 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,6 +1,8 @@ """ Helper functions for serializing (and deserializing) requests. """ +import inspect + from scrapy.http import Request from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -68,20 +70,6 @@ def request_from_dict(d, spider=None): ) -def _is_private_method(name): - return name.startswith('__') and not name.endswith('__') - - -def _mangle_private_name(obj, func, name): - qualname = getattr(func, '__qualname__', None) - if qualname is None: - classname = obj.__class__.__name__.lstrip('_') - return '_%s%s' % (classname, name) - else: - splits = qualname.split('.') - return '_%s%s' % (splits[-2], splits[-1]) - - def _find_method(obj, func): if obj: try: @@ -90,10 +78,17 @@ def _find_method(obj, func): pass else: if func_self is obj: - name = func.__func__.__name__ - if _is_private_method(name): - return _mangle_private_name(obj, func, name) - return name + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b8c140a7e..12c03d78e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): """ if include_headers: - include_headers = tuple(to_bytes(h.lower()) - for h in sorted(include_headers)) + include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) cache_key = (include_headers, keep_fragments) if cache_key not in cache: diff --git a/tests/spiders.py b/tests/spiders.py index 284c77829..33d5d02e1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, \ - 'All start requests consumed before any download happened' + assert self.seedsseen, 'All start requests consumed before any download happened' def parse(self, response): self.seedsseen.append(response.meta.get('seed')) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a56425b7..5ec5e2989 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,8 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'\ - .format(name=crawler.spider.exception_cls.__name__) + key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1a7be5c1b..46d85669c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -828,11 +828,15 @@ class S3TestCase(unittest.TestCase): def test_request_signing2(self): # puts an object into the johnsmith bucket. date = 'Tue, 27 Mar 2007 21:15:45 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ - 'Content-Type': 'image/jpeg', - 'Date': date, - 'Content-Length': '94328', - }) + req = Request( + 's3://johnsmith/photos/puppy.jpg', + method='PUT', + headers={ + 'Content-Type': 'image/jpeg', + 'Date': date, + 'Content-Length': '94328', + }, + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -912,11 +916,10 @@ class S3TestCase(unittest.TestCase): # ensure that spaces are quoted properly before signing date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request( - ("s3://johnsmith/photos/my puppy.jpg" - "?response-content-disposition=my puppy.jpg"), + "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg", method='GET', headers={'Date': date}, - ) + ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual( @@ -1096,8 +1099,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): self.assertEqual(response.text, 'A brief note') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "US-ASCII") request = Request("data:,A%20brief%20note") @@ -1106,8 +1108,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:;charset=iso-8859-7,%be%d3%be") @@ -1125,8 +1126,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): self.assertEqual(response.text, u'\u038e\u03a3\u038e') - self.assertEqual(type(response), - responsetypes.from_mimetype("text/plain")) + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 77b35a8c3..dbae4d3ae 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - assert isinstance(new, XmlResponse), \ - 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) def test_plain_response(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4d43e164f..87304d76c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -16,12 +16,12 @@ from w3lib.encoding import resolve_encoding SAMPLEDIR = join(tests_datadir, 'compressed') FORMAT = { - 'gzip': ('html-gzip.bin', 'gzip'), - 'x-gzip': ('html-gzip.bin', 'gzip'), - 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), - 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), - 'br': ('html-br.bin', 'br') - } + 'gzip': ('html-gzip.bin', 'gzip'), + 'x-gzip': ('html-gzip.bin', 'gzip'), + 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), + 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + 'br': ('html-br.bin', 'br'), +} class HttpCompressionTest(TestCase): @@ -40,12 +40,12 @@ class HttpCompressionTest(TestCase): body = sample.read() headers = { - 'Server': 'Yaws/1.49 Yet Another Web Server', - 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', - 'Content-Length': len(body), - 'Content-Type': 'text/html', - 'Content-Encoding': contentencoding, - } + 'Server': 'Yaws/1.49 Yet Another Web Server', + 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', + 'Content-Length': len(body), + 'Content-Type': 'text/html', + 'Content-Encoding': contentencoding, + } response = Response('http://scrapytest.org/', body=body, headers=headers) response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 3044c5b21..5f131834a 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -184,8 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase): rsp = Response(url, headers={'Location': url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) self.assertIs(r, rsp) - _test_passthrough(Request(url, meta={'handle_httpstatus_list': - [404, 301, 302]})) + _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]})) _test_passthrough(Request(url, meta={'handle_httpstatus_all': True})) def test_latin1_location(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 87d98d6ee..21d60b8bf 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -399,8 +399,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_bytes(self): data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} - r2 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'latin1') self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') @@ -408,8 +407,7 @@ class FormRequestTest(RequestTest): def test_custom_encoding_textual_data(self): data = {'price': u'£ 100'} - r3 = self.request_class("http://www.example.com", formdata=data, - encoding='latin1') + r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') @@ -469,7 +467,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html", encoding='latin1', - ) + ) req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 00568aee9..930a5dd99 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -46,13 +46,13 @@ class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', - SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' - ) + SCHEDULER_DEBUG=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', + ) super(MockCrawler, self).__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -305,10 +305,12 @@ class StartUrlsSpider(Spider): class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( - StartUrlsSpider, - {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', - 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'} - ) + spidercls=StartUrlsSpider, + settings_dict={ + 'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + }, + ) @defer.inlineCallbacks def tearDown(self): @@ -329,9 +331,9 @@ class TestIncompatibility(unittest.TestCase): def _incompatible(self): settings = dict( - SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', - CONCURRENT_REQUESTS_PER_IP=1 - ) + SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', + CONCURRENT_REQUESTS_PER_IP=1, + ) crawler = Crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name='spider') diff --git a/tests/test_selector.py b/tests/test_selector.py index 3629feb16..bcf653444 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -75,8 +75,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), - [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index dacd0147f..6b61df56f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() @@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 6291d1db8..be431e2d9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -514,32 +514,32 @@ class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) def test_valid_name_casevariants(self): for s, p in [ - (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), - (POLICY_NO_REFERRER, NoReferrerPolicy), - (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), - (POLICY_SAME_ORIGIN, SameOriginPolicy), - (POLICY_ORIGIN, OriginPolicy), - (POLICY_STRICT_ORIGIN, StrictOriginPolicy), - (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), - (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), - (POLICY_UNSAFE_URL, UnsafeUrlPolicy), - ]: + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2fac3da1f..363b015a8 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual( + body, + "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" + ) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 38630d012..2c3da3c05 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -167,7 +167,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_objtype_exception(self): i = self.xmliter(42, 'product') - self.assertRaises(AssertionError, next, i) + self.assertRaises(TypeError, next, i) def test_xmliter_encoding(self): body = ( diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c7572f02c..50b026d1c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -2,7 +2,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name +from scrapy.utils.reqser import request_to_dict, request_from_dict class RequestSerializationTest(unittest.TestCase): @@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_reference_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.parse_item_reference, + errback=self.spider.handle_error_reference) + self._assert_serializes_ok(r, spider=self.spider) + request_dict = request_to_dict(r, self.spider) + self.assertEqual(request_dict['callback'], 'parse_item_reference') + self.assertEqual(request_dict['errback'], 'handle_error_reference') + + def test_private_reference_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider._TestSpider__parse_item_reference, + errback=self.spider._TestSpider__handle_error_reference) + self._assert_serializes_ok(r, spider=self.spider) + request_dict = request_to_dict(r, self.spider) + self.assertEqual(request_dict['callback'], + '_TestSpider__parse_item_reference') + self.assertEqual(request_dict['errback'], + '_TestSpider__handle_error_reference') + def test_private_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider._TestSpider__parse_item_private, @@ -81,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) - def test_private_callback_name_matching(self): - self.assertTrue(_is_private_method('__a')) - self.assertTrue(_is_private_method('__a_')) - self.assertTrue(_is_private_method('__a_a')) - self.assertTrue(_is_private_method('__a_a_')) - self.assertTrue(_is_private_method('__a__a')) - self.assertTrue(_is_private_method('__a__a_')) - self.assertTrue(_is_private_method('__a___a')) - self.assertTrue(_is_private_method('__a___a_')) - self.assertTrue(_is_private_method('___a')) - self.assertTrue(_is_private_method('___a_')) - self.assertTrue(_is_private_method('___a_a')) - self.assertTrue(_is_private_method('___a_a_')) - self.assertTrue(_is_private_method('____a_a_')) - - self.assertFalse(_is_private_method('_a')) - self.assertFalse(_is_private_method('_a_')) - self.assertFalse(_is_private_method('__a__')) - self.assertFalse(_is_private_method('__')) - self.assertFalse(_is_private_method('___')) - self.assertFalse(_is_private_method('____')) - - def _assert_mangles_to(self, obj, name): - func = getattr(obj, name) - self.assertEqual( - _mangle_private_name(obj, func, func.__name__), - name - ) - - def test_private_name_mangling(self): - self._assert_mangles_to( - self.spider, '_TestSpider__parse_item_private') - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') - def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -125,14 +110,49 @@ class RequestSerializationTest(unittest.TestCase): r = Request("http://www.example.com", callback=self.spider.parse_item) self.assertRaises(ValueError, request_to_dict, r) + def test_unserializable_callback3(self): + """Parser method is removed or replaced dynamically.""" + + class MySpider(Spider): + + name = 'my_spider' + + def parse(self, response): + pass + + spider = MySpider() + r = Request("http://www.example.com", callback=spider.parse) + setattr(spider, 'parse', None) + self.assertRaises(ValueError, request_to_dict, r, spider=spider) + class TestSpiderMixin: def __mixin_callback(self, response): pass +def parse_item(response): + pass + + +def handle_error(failure): + pass + + +def private_parse_item(response): + pass + + +def private_handle_error(failure): + pass + + class TestSpider(Spider, TestSpiderMixin): name = 'test' + parse_item_reference = parse_item + handle_error_reference = handle_error + __parse_item_reference = private_parse_item + __handle_error_reference = private_handle_error def parse_item(self, response): pass diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 7fb0e1f1e..3ab0370d7 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -80,114 +80,124 @@ class UrlUtilsTest(unittest.TestCase): class AddHttpIfNoScheme(unittest.TestCase): def test_add_scheme(self): - self.assertEqual(add_http_if_no_scheme('www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('example.com'), - 'http://example.com') + self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') def test_path(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_port(self): - self.assertEqual(add_http_if_no_scheme('www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('www.example.com:80'), + 'http://www.example.com:80') def test_fragment(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_query(self): - self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_username_password(self): - self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('username:password@www.example.com'), + 'http://username:password@www.example.com') def test_complete_url(self): self.assertEqual( add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' - ) + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_http(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com'), - 'http://www.example.com') + self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') def test_preserve_http_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('http://example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('http://example.com'), + 'http://example.com') def test_preserve_http_path(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_preserve_http_port(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com:80'), + 'http://www.example.com:80') def test_preserve_http_fragment(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_preserve_http_query(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_preserve_http_username_password(self): - self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('http://username:password@www.example.com'), + 'http://username:password@www.example.com') def test_preserve_http_complete_url(self): self.assertEqual( add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' - ) + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_protocol_relative(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com'), - 'http://www.example.com') + self.assertEqual( + add_http_if_no_scheme('//www.example.com'), 'http://www.example.com') def test_protocol_relative_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('//example.com'), - 'http://example.com') + self.assertEqual( + add_http_if_no_scheme('//example.com'), 'http://example.com') def test_protocol_relative_path(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') def test_protocol_relative_port(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com:80'), - 'http://www.example.com:80') + self.assertEqual( + add_http_if_no_scheme('//www.example.com:80'), + 'http://www.example.com:80') def test_protocol_relative_fragment(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') def test_protocol_relative_query(self): - self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual( + add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') def test_protocol_relative_username_password(self): - self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'), - 'http://username:password@www.example.com') + self.assertEqual( + add_http_if_no_scheme('//username:password@www.example.com'), + 'http://username:password@www.example.com') def test_protocol_relative_complete_url(self): self.assertEqual( add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag' - ) + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') def test_preserve_https(self): - self.assertEqual(add_http_if_no_scheme('https://www.example.com'), - 'https://www.example.com') + self.assertEqual( + add_http_if_no_scheme('https://www.example.com'), + 'https://www.example.com') def test_preserve_ftp(self): - self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), - 'ftp://www.example.com') + self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com') class GuessSchemeTest(unittest.TestCase): @@ -211,41 +221,49 @@ def create_skipped_scheme_t(args): return do_expected -for k, args in enumerate([ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), +for k, args in enumerate( + [ + ('/index', 'file://'), + ('/index.html', 'file://'), + ('./index.html', 'file://'), + ('../index.html', 'file://'), + ('../../index.html', 'file://'), + ('./data/index.html', 'file://'), + ('.hidden/data/index.html', 'file://'), + ('/home/user/www/index.html', 'file://'), + ('//home/user/www/index.html', 'file://'), + ('file:///home/user/www/index.html', 'file://'), - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), + ('index.html', 'http://'), + ('example.com', 'http://'), + ('www.example.com', 'http://'), + ('www.example.com/index.html', 'http://'), + ('http://example.com', 'http://'), + ('http://example.com/index.html', 'http://'), + ('localhost', 'http://'), + ('localhost/index.html', 'http://'), - # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), - - ], start=1): + # some corner cases (default to http://) + ('/', 'http://'), + ('.../test', 'http://'), + ], + start=1, +): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate([ - (r'C:\absolute\path\to\a\file.html', 'file://', - 'Windows filepath are not supported for scrapy shell'), - ], start=1): +for k, args in enumerate( + [ + ( + r'C:\absolute\path\to\a\file.html', + 'file://', + 'Windows filepath are not supported for scrapy shell', + ), + ], + start=1, +): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k setattr(GuessSchemeTest, t_method.__name__, t_method) @@ -281,7 +299,7 @@ class StripUrl(unittest.TestCase): ('http://www.example.com', True, 'http://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) def test_credentials(self): @@ -294,7 +312,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_credentials_encoded_delims(self): @@ -313,7 +331,7 @@ class StripUrl(unittest.TestCase): # password: "user@domain.com" ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', 'ftp://www.example.com/index.html?somekey=somevalue'), - ]: + ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_default_ports_creds_off(self): @@ -341,7 +359,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i), o) def test_default_ports(self): @@ -369,7 +387,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): @@ -397,7 +415,7 @@ class StripUrl(unittest.TestCase): ('ftp://username:password@www.example.com:221/file.txt', 'ftp://username:password@www.example.com:221/file.txt'), - ]: + ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): @@ -413,7 +431,7 @@ class StripUrl(unittest.TestCase): ('https://username:password@www.example.com:443/index.html', 'https://www.example.com/'), - ]: + ]: self.assertEqual(strip_url(i, origin_only=True), o) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4abebbfb..de61e2125 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -149,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers={ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - })) + }, + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" @@ -165,7 +166,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): headers=Headers({ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - }))) + }), + )) self._test(factory, b"GET /bar HTTP/1.0\r\n" diff --git a/tox.ini b/tox.ini index cd118c921..697328ebd 100644 --- a/tox.ini +++ b/tox.ini @@ -102,16 +102,6 @@ setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck -[asyncio] +[testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio - -[testenv:py35-asyncio] -basepython = python3.5 -deps = {[testenv]deps} -commands = {[asyncio]commands} - -[testenv:py38-asyncio] -basepython = python3.8 -deps = {[testenv]deps} -commands = {[asyncio]commands}