diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80df9469d..98fa44c7f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,6 @@ jobs: - python-version: 3.8 env: TOXENV: pylint - TOX_PIP_VERSION: 20.3.3 - python-version: 3.6 env: TOXENV: typing @@ -38,8 +37,5 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ea50e644..1fc8d914b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -46,7 +46,6 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps - TOX_PIP_VERSION: 20.3.3 steps: - uses: actions/checkout@v2 @@ -73,9 +72,6 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6fabf5cde..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,9 @@ jobs: - python-version: "3.10" env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/README.rst b/README.rst index 05f10bb6c..6b563d638 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ -.. image:: /artwork/scrapy-logo.jpg - :width: 400px +.. image:: https://scrapy.org/img/scrapylogo.png ====== Scrapy diff --git a/docs/conf.py b/docs/conf.py index 406c4d94a..d5e139e66 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions # Never documented before, and deprecated now. - r'^scrapy\.item\.DictItem$', r'^scrapy\.linkextractors\.FilteringLinkExtractor$', # Implementation detail of LxmlLinkExtractor diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index 96363c7d5..edff894c6 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -7,7 +7,7 @@ Examples The best way to learn is with examples, and Scrapy is no exception. For this reason, there is an example Scrapy project named quotesbot_, that you can use to play and learn more about Scrapy. It contains two spiders for -http://quotes.toscrape.com, one using CSS selectors and another one using XPath +https://quotes.toscrape.com, one using CSS selectors and another one using XPath expressions. The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8581dde0b..b8d3a16bc 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -190,7 +190,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python, install a new, updated version +* *(Recommended)* **Don't** use system Python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: @@ -234,8 +234,8 @@ For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependencies will be built during installation. -On macOS, you are likely to face an issue with building Cryptography dependency, -solution to this problem is described +On macOS, you are likely to face an issue with building the Cryptography +dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command recommends (only needed when installing Scrapy). Installing on Linux has no special diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 405bf845d..f3d652621 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -http://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination:: import scrapy @@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): name = 'quotes' start_urls = [ - 'http://quotes.toscrape.com/tag/humor/', + 'https://quotes.toscrape.com/tag/humor/', ] def parse(self, response): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ca5856881..5697b9608 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to scrape `quotes.toscrape.com `_, a website +We are going to scrape `quotes.toscrape.com `_, a website that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -143,9 +143,9 @@ similar to this:: 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) @@ -184,8 +184,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -207,7 +207,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the :ref:`Scrapy shell `. Run:: - scrapy shell 'http://quotes.toscrape.com/page/1/' + scrapy shell 'https://quotes.toscrape.com/page/1/' .. note:: @@ -217,18 +217,18 @@ using the :ref:`Scrapy shell `. Run:: On Windows, use double quotes instead:: - scrapy shell "http://quotes.toscrape.com/page/1/" + scrapy shell "https://quotes.toscrape.com/page/1/" You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request - [s] response <200 http://quotes.toscrape.com/page/1/> + [s] request + [s] response <200 https://quotes.toscrape.com/page/1/> [s] settings [s] spider [s] Useful shortcuts: @@ -241,7 +241,7 @@ object: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') >>> response.css('title') [] @@ -355,7 +355,7 @@ Extracting quotes and authors Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the web page. -Each quote in http://quotes.toscrape.com is represented by HTML elements that look +Each quote in https://quotes.toscrape.com is represented by HTML elements that look like this: .. code-block:: html @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'http://quotes.toscrape.com' + $ scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: @@ -444,8 +444,8 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -458,9 +458,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} @@ -505,7 +505,7 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from http://quotes.toscrape.com, you want quotes from all the pages in the website. +from https://quotes.toscrape.com, you want quotes from all the pages in the website. Now that you know how to extract data from pages, let's see how to follow links from them. @@ -549,7 +549,7 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -654,7 +654,7 @@ this time for scraping author information:: class AuthorSpider(scrapy.Spider): name = 'author' - start_urls = ['http://quotes.toscrape.com/'] + start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): author_page_links = response.css('.author + a') @@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'http://quotes.toscrape.com/' + url = 'https://quotes.toscrape.com/' tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag @@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument:: If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as -``http://quotes.toscrape.com/tag/humor``. +``https://quotes.toscrape.com/tag/humor``. You can :ref:`learn more about handling spider arguments here `. diff --git a/docs/news.rst b/docs/news.rst index 509366c17..47a808693 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,3 +1,25 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes @@ -4358,7 +4380,7 @@ Code rearranged and removed - Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 28241ae24..8712d4268 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. - .. _install-asyncio: Installing the asyncio reactor @@ -41,6 +36,34 @@ use it instead of the default asyncio event loop. .. _asyncio-await-dfd: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + Awaiting on Deferreds ===================== diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 057b1ec62..96475899f 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out. -First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: +First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal: .. code-block:: none - $ scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "https://quotes.toscrape.com/" Then, back to your web browser, right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/', 'quotes.html') + response = load_response('https://quotes.toscrape.com/', 'quotes.html') >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] @@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the type ``json``. If we click on this request, we see that the request URL is -``http://quotes.toscrape.com/api/quotes?page=1`` and the response +``https://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click on the request and open ``Open in new tab`` to get a better overview. @@ -247,7 +247,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) @@ -255,7 +255,7 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" + url = f"https://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) This spider starts at the first page of the quotes-API. With each @@ -280,7 +280,7 @@ request:: from scrapy import Request request = Request.from_curl( - "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" @@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools -.. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll -.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _quotes.toscrape.com: https://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll +.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 116967280..7994027d2 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -322,7 +322,7 @@ Post-Processing Scrapy provides an option to activate plugins to post-process feeds before they are exported to feed storages. In addition to using :ref:`builtin plugins `, you -can create your own :ref:`plugins `. +can create your own :ref:`plugins `. These plugins can be activated through the ``postprocessing`` option of a feed. The option must be passed a list of post-processing plugins in the order you want @@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods: Close the target file object. -To pass a parameter to your plugin, use :ref:`feed options `. You +To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. @@ -744,6 +744,9 @@ The function signature should be as follows: :param spider: source spider of the feed items :type spider: scrapy.Spider + .. caution:: The function should return a new dictionary, modifying + the received ``params`` in-place is deprecated. + For example, to include the :attr:`name ` of the source spider in the feed URI: diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index d593c74c6..3bf23d5f5 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many HTTP 404 and 500 responses, and you want to hide all messages like this:: 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring - response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code is not handled or not allowed The first thing to note is a logger name - it is in brackets: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..1a9d56143 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 42ce22158..e0435e901 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -950,6 +950,14 @@ TextResponse objects Returns a Python object from deserialized JSON document. The result is cached after the first call. + .. method:: TextResponse.urljoin(url) + + Constructs an absolute url by combining the Response's base url with + a possible relative url. The base url shall be extracted from the + ```` tag, or just the Response's :attr:`url` if there is no such + tag. + + HtmlResponse objects -------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 210c1def7..4e105642d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy:: def start_requests(self): reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes:: from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1638,10 +1638,9 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 63ad3a9ad..2fbd0b51c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,7 +60,7 @@ Let's take an example:: class SignalSpider(scrapy.Spider): name = 'signals' - start_urls = ['http://quotes.toscrape.com/page/1/'] + start_urls = ['https://quotes.toscrape.com/page/1/'] @classmethod def from_crawler(cls, crawler, *args, **kwargs): diff --git a/pylintrc b/pylintrc index 699686e16..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -112,6 +112,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/pytest.ini b/pytest.ini index fa5d6b34f..ae2ed2029 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,5 +21,3 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed -filterwarnings= - ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 91482ce01..491c4beab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,13 +1,13 @@ import sys import os -import optparse +import argparse import cProfile import inspect import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -123,8 +123,6 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) sys.exit(0) @@ -133,12 +131,14 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = f"scrapy {cmdname} {cmd.syntax()}" - parser.description = cmd.long_desc() + parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) + opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..fb304b8c0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,7 +2,7 @@ Base class for Scrapy commands """ import os -from optparse import OptionGroup +import argparse from typing import Any, Dict from twisted.python import failure @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() @@ -59,22 +59,20 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_option("--nolog", action="store_true", - help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") - - parser.add_option_group(group) + group = parser.add_argument_group(title='Global Options') + group.add_argument("--logfile", metavar="FILE", + help="log file. if omitted stderr will be used") + group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})") + group.add_argument("--nolog", action="store_true", + help="disable logging completely") + group.add_argument("--profile", metavar="FILE", default=None, + help="write python cProfile stats to FILE") + group.add_argument("--pidfile", metavar="FILE", + help="write process ID to FILE") + group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", + help="set/override setting (may be repeated)") + group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: @@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand): """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") - parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_argument("-o", "--output", metavar="FILE", action="append", + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") + parser.add_argument("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.overwrite_output, ) self.settings.set('FEEDS', feeds, priority='cmdline') + + +class ScrapyHelpFormatter(argparse.HelpFormatter): + """ + Help Formatter for scrapy command line help messages. + """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + super().__init__(prog, indent_increment=indent_increment, + max_help_position=max_help_position, width=width) + + def _join_parts(self, part_strings): + parts = self.format_part_strings(part_strings) + return super()._join_parts(parts) + + def format_part_strings(self, part_strings): + """ + Underline and title case command line help message headers. + """ + if part_strings and part_strings[0].startswith("usage: "): + part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] + headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + for index in headings[::-1]: + char = '-' if "Global Options" in part_strings[index] else '=' + part_strings[index] = part_strings[index][:-2].title() + underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + part_strings.insert(index + 1, underline) + return part_strings diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ae21d86e6..a16f4beb7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -49,10 +49,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', + help="print contract tests for all spiders") def run(self, args, opts): # load contracts diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 95f87e8c3..9b2ebb37f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -26,11 +26,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2082a4974..ed5f588e9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -44,16 +44,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_argument("-e", "--edit", dest="edit", action="store_true", + help="Edit spider after creating it") + parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") + parser.add_argument("-t", "--template", dest="template", default="basic", + help="Uses a custom template.") + parser.add_argument("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 52118db1b..a3f6b96f4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument("--spider", dest="spider", default=None, + help="use this spider without looking for one") + parser.add_argument("--pipelines", action="store_true", + help="process items through pipelines") + parser.add_argument("--nolinks", dest="nolinks", action="store_true", + help="don't show links to follow (extracted requests)") + parser.add_argument("--noitems", dest="noitems", action="store_true", + help="don't show scraped items") + parser.add_argument("--nocolour", dest="nocolour", action="store_true", + help="avoid using pygments to colorize the output") + parser.add_argument("-r", "--rules", dest="rules", action="store_true", + help="use CrawlSpider rules to discover the callback") + parser.add_argument("-c", "--callback", dest="callback", + help="use this callback for parsing, instead looking for a callback") + parser.add_argument("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_argument("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") + parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, + help="maximum depth for parsing requests [default: %default]") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 8d49e440f..1b2e2601e 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -18,16 +18,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument("--get", dest="get", metavar="SETTING", + help="print raw setting value") + parser.add_argument("--getbool", dest="getbool", metavar="SETTING", + help="print setting value, interpreted as a boolean") + parser.add_argument("--getint", dest="getint", metavar="SETTING", + help="print setting value, interpreted as an integer") + parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", + help="print setting value, interpreted as a float") + parser.add_argument("--getlist", dest="getlist", metavar="SETTING", + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..f67a5886a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -33,12 +33,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + parser.add_argument("--spider", dest="spider", + help="use this spider") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1237610cb..c6a3c273a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -16,8 +16,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index c8f873334..b1f52abe2 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,3 +1,4 @@ +import argparse from scrapy.commands import fetch from scrapy.utils.response import open_in_browser @@ -12,7 +13,7 @@ class Command(fetch.Command): def add_options(self, parser): super().add_options(parser) - parser.remove_option("--headers") + parser.add_argument('--headers', help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,20 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + reactor_class = self.settings.get("TWISTED_REACTOR") + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +168,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +261,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() @@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 5479cd0e2..25fb1ed9d 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,7 +1,22 @@ from scrapy.exceptions import NotConfigured +from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from scrapy.utils.response import response_httprepr -from scrapy.utils.python import global_object_name + +from twisted.web import http + + +def get_header_size(headers): + size = 0 + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + for v in value: + size += len(b": ") + len(key) + len(v) + return size + len(b'\r\n') * (len(headers.keys()) - 1) + + +def get_status_size(response_status): + return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: @@ -24,7 +39,8 @@ class DownloaderStats: def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response_httprepr(response)) + reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + # response.body + b"\r\n"+ response.header + b"\r\n" + response.status self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 36cca2d05..1c26e81db 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, _BaseItem): + if isinstance(value, Item): return self.export_item(value) elif is_item(value): return dict(self._serialize_item(value)) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 370723368..e7097b7a1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,14 +11,14 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile -from typing import Any, Optional, Tuple +from typing import Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from zope.interface import implementer, Interface -from scrapy import signals +from scrapy import signals, Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available @@ -524,7 +524,12 @@ class FeedExporter: raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance - def _get_uri_params(self, spider, uri_params, slot=None): + def _get_uri_params( + self, + spider: Spider, + uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + slot: Optional[_FeedSlot] = None, + ) -> dict: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -532,9 +537,18 @@ class FeedExporter: params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = load_object(uri_params) if uri_params else lambda x, y: None - uripar_function(params, spider) - return params + original_params = params.copy() + uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + new_params = uripar_function(params, spider) + if new_params is None or original_params != params: + warnings.warn( + 'Modifying the params dictionary in-place in the function defined in ' + 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' + 'setting is deprecated. The function must return a new dictionary ' + 'instead.', + category=ScrapyDeprecationWarning + ) + return new_params if new_params is not None else params def _load_filter(self, feed_options): # load the item filter if declared else load the default filter class diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 27bd55c07..89516b9b6 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst """ import json -import warnings from contextlib import suppress from typing import Generator, Tuple from urllib.parse import urljoin @@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -66,13 +64,6 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def body_as_unicode(self): - """Return body as unicode""" - warnings.warn('Response.body_as_unicode() is deprecated, ' - 'please use Response.text instead.', - ScrapyDeprecationWarning, stacklevel=2) - return self.text - def json(self): """ .. versionadded:: 2.2 diff --git a/scrapy/item.py b/scrapy/item.py index 2ccd7ad18..2521ac829 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -9,45 +9,15 @@ from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from typing import Dict -from warnings import warn -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class _BaseItem(object_ref): - """ - Temporary class used internally to avoid the deprecation - warning raised by isinstance checks using BaseItem. - """ - pass - - -class _BaseItemMeta(ABCMeta): - def __instancecheck__(cls, instance): - if cls is BaseItem: - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__instancecheck__(instance) - - -class BaseItem(_BaseItem, metaclass=_BaseItemMeta): - """ - Deprecated, please use :class:`scrapy.item.Item` instead - """ - - def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) - - class Field(dict): """Container of field metadata""" -class ItemMeta(_BaseItemMeta): +class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses @@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, BaseItem): +class Item(MutableMapping, object_ref, metaclass=ItemMeta): + """ + Base class for scraped items. - fields: Dict[str, Field] = {} + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines `. - def __new__(cls, *args, **kwargs): - if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields ` for additional information. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked ` to debug memory leaks. + """ + + fields: Dict[str, Field] def __init__(self, *args, **kwargs): self._values = {} @@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem): def __iter__(self): return iter(self._values) - __hash__ = BaseItem.__hash__ + __hash__ = object_ref.__hash__ def keys(self): return self._values.keys() @@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem): """Return a :func:`~copy.deepcopy` of this item. """ return deepcopy(self) - - -class Item(DictItem, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,12 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) - - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..6b1ad0828 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -375,9 +375,13 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items()} + def _get_key(self, key_value): + return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value)) + def copy_to_dict(self): """ Make a copy of current settings and convert to a dict. diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index bef2d6b24..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24873f75d..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0441c0358..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -152,6 +152,9 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 11c4206c2..1221b39b2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,11 +14,11 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.deprecate import ScrapyDeprecationWarning -_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes def arg_to_iter(arg): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..741dce350 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -13,6 +14,7 @@ from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.decorators import deprecated from w3lib import html @@ -50,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f'{status_int} {to_unicode(message)}' +@deprecated def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes @@ -80,8 +83,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..3f219098c --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(SelectReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..1f5a44010 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/mockserver.py b/tests/mockserver.py index ab9aec6a6..72d7e0241 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web import resource, server from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File -from twisted.web.test.test_webclient import PayloadResource from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode @@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None): return default -class LeafResource(Resource): +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class ErrorResource(resource.Resource): + def render(self, request): + request.setResponseCode(401) + if request.args.get(b"showlength"): + request.setHeader(b"content-length", b"0") + return b"" + + +class NoLengthResource(resource.Resource): + def render(self, request): + return b"nolength" + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + # only sends 3 bytes even though it claims to send 5 + request.setHeader(b"content-length", b"5") + request.write(b"abc") + return b"" + + +class LeafResource(resource.Resource): isLeaf = True @@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() -class Root(Resource): +class Root(resource.Resource): def __init__(self): - Resource.__init__(self) + resource.Resource.__init__(self) self.putChild(b"status", Status()) self.putChild(b"follow", Follow()) self.putChild(b"delay", Delay()) @@ -187,7 +249,7 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 591075a98..8233e0101 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase): settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) + + def test_pathlib_path_as_feeds_key(self): + self.assertEqual(self._execute('settings', '--get', 'FEEDS'), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 8a719ddf2..b0ac6e98b 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,5 +1,14 @@ +from pathlib import Path + EXTENSIONS = { 'tests.test_cmdline.extensions.TestExtension': 0, } TEST1 = 'default' + +FEEDS = { + Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, +} diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 34f5e59dd..c3d705194 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -19,11 +19,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://example.com'] + start_urls = ['http://toscrape.com'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://example.com + @url http://toscrape.com {contracts} \"\"\" {parse_def} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index ed3848d88..f21ee971d 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,9 @@ import os +import argparse from os.path import join, abspath, isfile, exists from twisted.internet import defer +from scrapy.commands import parse +from scrapy.settings import Settings from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode @@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' with open(file_path, 'r') as f: self.assertEqual(f.read(), content) + + def test_parse_add_options(self): + command = parse.Command() + command.settings = Settings() + parser = argparse.ArgumentParser( + prog='scrapy', formatter_class=argparse.HelpFormatter, + conflict_handler='resolve', prefix_chars='-' + ) + command.add_options(parser) + namespace = parser.parse_args( + ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ) + self.assertTrue(namespace.nolinks) + self.assertEqual(namespace.depth, 2) + self.assertEqual(namespace.spider, self.spider_name) + self.assertTrue(namespace.verbose) diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..7cd19b29a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,6 +1,6 @@ import inspect import json -import optparse +import argparse import os import platform import re @@ -23,7 +23,7 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import ScrapyCommand +from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = optparse.OptionParser( - formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve', - ) + self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + def test_help_formatter(self): + formatter = ScrapyHelpFormatter(prog='scrapy') + part_strings = ['usage: scrapy genspider [options] \n\n', + '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + self.assertEqual( + formatter._join_parts(part_strings), + ('Usage\n=====\n scrapy genspider [options] \n\n\n' + 'Optional Arguments\n==================\n\n' + 'Global Options\n--------------\n') + ) + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -674,9 +683,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -699,15 +705,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): @@ -765,6 +771,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -777,35 +784,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() @@ -822,6 +821,21 @@ class BenchCommandTest(CommandTest): self.assertNotIn('Unhandled Error', log) +class ViewCommandTest(CommandTest): + + def test_methods(self): + command = view.Command() + command.settings = Settings() + parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', + formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') + command.add_options(parser) + self.assertEqual(command.short_desc(), + "Open URL in browser, as seen by Scrapy") + self.assertIn("URL using the Scrapy downloader and show its", + command.long_desc()) + + class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..8f6227109 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,10 +4,8 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version @@ -271,6 +269,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -279,35 +278,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class ScriptRunnerMixin: @@ -328,17 +302,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -377,14 +345,26 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') @@ -404,9 +384,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..2bb53950d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -14,8 +15,6 @@ from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource, - NoLengthResource, PayloadResource) from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -33,7 +32,15 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) from tests.spiders import SingleRequestSpider @@ -287,6 +294,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +309,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1073,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 1f2616e35..9e75f0a50 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,8 +1,10 @@ +from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual('downloader/response_count', 1) + def test_response_len(self): + body = (b'', b'not_empty') # empty/notempty body + headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + test_responses = [ # form test responses with all combinations of body/headers + Response( + url='scrapytest.org', + status=200, + body=r[0], + headers=r[1] + ) + for r in product(body, headers) + ] + for test_response in test_responses: + self.crawler.stats.set_value('downloader/response_bytes', 0) + self.mw.process_response(self.req, test_response, self.spider) + self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual('downloader/exception_count', 1) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253f3119c..f0acf1941 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): ), ) ) + + +class URIParamsTest: + + spider_name = "uri_params_spider" + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + raise NotImplementedError + + def test_default(self): + settings = self.build_settings( + uri='file:///tmp/%(name)s', + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + 'Modifying the params dictionary in-place in the ' + 'function defined in the FEED_URI_PARAMS setting or ' + 'in the uri_params key of the FEEDS setting is ' + 'deprecated. The function must return a new ' + 'dictionary instead.' + ), + ) + ) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(KeyError): + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, 'foo': self.spider_name} + + settings = self.build_settings( + uri='file:///tmp/%(foo)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + +class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + extra_settings = {} + if uri_params: + extra_settings['FEED_URI_PARAMS'] = uri_params + return { + 'FEED_URI': uri, + **extra_settings, + } + + +class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + options = { + 'format': 'jl', + } + if uri_params: + options['uri_params'] = uri_params + return { + 'FEEDS': { + uri: options, + }, + } diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0ec5257e1..2986f884f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,10 +1,8 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest): original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') - # check body_as_unicode - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) - # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) @@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') - def test_body_as_unicode_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), 'Hello') - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) diff --git a/tests/test_item.py b/tests/test_item.py index c94bb44af..25f2aea0a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,7 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] - def test_dictitem_deprecation_warning(self): - """Make sure the DictItem deprecation warning is not issued for - Item""" - with catch_warnings(record=True) as warnings: - Item() - self.assertEqual(len(warnings), 0) - - class SubclassedItem(Item): - pass - SubclassedItem() - self.assertEqual(len(warnings), 0) - class ItemMetaTest(unittest.TestCase): @@ -303,94 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase): super().__init__(*args, **kwargs) -class DictItemTest(unittest.TestCase): - - def test_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - DictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedDictItem(DictItem): - pass - SubclassedDictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class BaseItemTest(unittest.TestCase): - - def test_isinstance_check(self): - - class SubclassedBaseItem(BaseItem): - pass - - class SubclassedItem(Item): - pass - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) - - def test_deprecation_warning(self): - """ - Make sure deprecation warnings are logged whenever BaseItem is used, - either instantiated or in an isinstance check - """ - with catch_warnings(record=True) as warnings: - BaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - - class SubclassedBaseItem(BaseItem): - pass - - SubclassedBaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertFalse(isinstance("foo", BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): - """ - Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. - """ - class SubclassedItem(Item): - pass - - with catch_warnings(record=True) as warnings: - Item() - SubclassedItem() - _BaseItem() - self.assertFalse(isinstance("foo", _BaseItem)) - self.assertFalse(isinstance("foo", Item)) - self.assertFalse(isinstance("foo", SubclassedItem)) - self.assertTrue(isinstance(_BaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), Item)) - self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) - self.assertEqual(len(warnings), 0) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_spider.py b/tests/test_spider.py index a7c3ee048..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 6e4cb9b6e..a6d55cb38 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -21,14 +21,6 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks -from twisted.web.test.test_webclient import ( - ForeverTakingResource, - ErrorResource, - NoLengthResource, - HostHeaderResource, - PayloadResource, - BrokenDownloadResource, -) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -36,7 +28,15 @@ from scrapy.http import Request, Headers from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory +from tests.mockserver import ( + BrokenDownloadResource, + ErrorResource, + ForeverTakingResource, + HostHeaderResource, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): diff --git a/tox.ini b/tox.ini index e4514f512..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.1 commands = pylint conftest.py docs extras scrapy setup.py tests