+ “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+ by Albert Einstein
+ (about)
+
+
+ “There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+ by Albert Einstein
+ (about)
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index a54a6bbe9..914d1d05f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,8 +27,13 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
+ 'hoverxref.extension',
+ 'notfound.extension',
'scrapydocs',
- 'sphinx.ext.autodoc'
+ 'sphinx.ext.autodoc',
+ 'sphinx.ext.coverage',
+ 'sphinx.ext.intersphinx',
+ 'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
@@ -73,6 +78,8 @@ language = 'en'
# List of documents that shouldn't be included in the build.
#unused_docs = []
+exclude_patterns = ['build']
+
# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['.build']
@@ -218,3 +225,66 @@ linkcheck_ignore = [
'http://localhost:\d+', 'http://hg.scrapy.org',
'http://directory.google.com/'
]
+
+
+# Options for the Coverage extension
+# ----------------------------------
+coverage_ignore_pyobjects = [
+ # Contract’s add_pre_hook and add_post_hook are not documented because
+ # they should be transparent to contract developers, for whom pre_hook and
+ # post_hook should be the actual concern.
+ r'\bContract\.add_(pre|post)_hook$',
+
+ # ContractsManager is an internal class, developers are not expected to
+ # interact with it directly in any way.
+ r'\bContractsManager\b$',
+
+ # For default contracts we only want to document their general purpose in
+ # their __init__ method, the methods they reimplement to achieve that purpose
+ # should be irrelevant to developers using those contracts.
+ r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
+
+ # Methods of downloader middlewares are not documented, only the classes
+ # themselves, since downloader middlewares are controlled through Scrapy
+ # settings.
+ r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
+
+ # Base classes of downloader middlewares are implementation details that
+ # are not meant for users.
+ r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
+
+ # Private exception used by the command-line interface implementation.
+ r'^scrapy\.exceptions\.UsageError',
+
+ # Methods of BaseItemExporter subclasses are only documented in
+ # BaseItemExporter.
+ r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
+
+ # Extension behavior is only modified through settings. Methods of
+ # extension classes, as well as helper functions, are implementation
+ # details that are not documented.
+ r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods
+ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions
+
+ # Never documented before, and deprecated now.
+ r'^scrapy\.item\.DictItem$',
+]
+
+
+# Options for the InterSphinx extension
+# -------------------------------------
+
+intersphinx_mapping = {
+ 'coverage': ('https://coverage.readthedocs.io/en/stable', None),
+ 'pytest': ('https://docs.pytest.org/en/latest', None),
+ 'python': ('https://docs.python.org/3', None),
+ 'sphinx': ('https://www.sphinx-doc.org/en/master', None),
+ 'tox': ('https://tox.readthedocs.io/en/latest', None),
+ 'twisted': ('https://twistedmatrix.com/documents/current', None),
+}
+
+
+# Options for sphinx-hoverxref options
+# ------------------------------------
+
+hoverxref_auto_ref = True
diff --git a/docs/conftest.py b/docs/conftest.py
new file mode 100644
index 000000000..8c735e838
--- /dev/null
+++ b/docs/conftest.py
@@ -0,0 +1,29 @@
+import os
+from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
+
+from scrapy.http.response.html import HtmlResponse
+from sybil import Sybil
+from sybil.parsers.codeblock import CodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+from sybil.parsers.skip import skip
+
+
+def load_response(url, filename):
+ input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
+ with open(input_path, 'rb') as input_file:
+ return HtmlResponse(url, body=input_file.read())
+
+
+def setup(namespace):
+ namespace['load_response'] = load_response
+
+
+pytest_collect_file = Sybil(
+ parsers=[
+ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
+ CodeBlockParser(future_imports=['print_function']),
+ skip,
+ ],
+ pattern='*.rst',
+ setup=setup,
+).pytest()
diff --git a/docs/contributing.rst b/docs/contributing.rst
index cf27337c8..234c4bcee 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -55,7 +55,7 @@ guidelines when you're going to report a new bug.
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
been discussed there, or if you're not sure if what you're seeing is a bug.
- You can also ask in the `#scrapy` IRC channel.
+ You can also ask in the ``#scrapy`` IRC channel.
* write **complete, reproducible, specific bug reports**. The smaller the test
case, the better. Remember that other developers won't have your project to
@@ -99,6 +99,15 @@ Well-written patches should:
the documentation changes in the same patch. See `Documentation policies`_
below.
+* if you're adding a private API, please add a regular expression to the
+ ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
+ private API from documentation coverage checks.
+
+ To see if your private API is skipped properly, generate a documentation
+ coverage report as follows::
+
+ tox -e docs-coverage
+
.. _submitting-patches:
Submitting patches
@@ -162,34 +171,39 @@ Scrapy:
See https://help.github.com/articles/setting-your-username-in-git/ for
setup instructions.
+.. _documentation-policies:
+
Documentation policies
======================
-* **Don't** use docstrings for documenting classes, or methods which are
- already documented in the official (sphinx) documentation. Alternatively,
- **do** provide a docstring, but make sure sphinx documentation uses
- autodoc_ extension to pull the docstring. For example, the
- :meth:`ItemLoader.add_value` method should be either
- documented only in the sphinx documentation (not as a docstring), or
- it should have a docstring which is pulled to sphinx documentation using
- autodoc_ extension.
+For reference documentation of API members (classes, methods, etc.) use
+docstrings and make sure that the Sphinx documentation uses the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
+documentation should follow docstring conventions (`PEP 257`_) and be
+IDE-friendly: short, to the point, and it may provide short examples.
-* **Do** use docstrings for documenting functions not present in the official
- (sphinx) documentation, such as functions from ``scrapy.utils`` package and
- its sub-modules.
+Other types of documentation, such as tutorials or topics, should be covered in
+files within the ``docs/`` directory. This includes documentation that is
+specific to an API member, but goes beyond API reference documentation.
-.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+In any case, if something is covered in a docstring, use the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
+documentation instead of duplicating the docstring in files within the
+``docs/`` directory.
Tests
=====
-Tests are implemented using the `Twisted unit-testing framework`_, running
-tests requires `tox`_.
+Tests are implemented using the :doc:`Twisted unit-testing framework
+`. Running tests requires
+:doc:`tox `.
+
+.. _running-tests:
Running tests
-------------
-Make sure you have a recent enough `tox`_ installation:
+Make sure you have a recent enough :doc:`tox ` installation:
``tox --version``
@@ -205,14 +219,37 @@ To run a specific test (say ``tests/test_loader.py``) use:
``tox -- tests/test_loader.py``
-To see coverage report install `coverage`_ (``pip install coverage``) and run:
+To run the tests on a specific :doc:`tox ` environment, use
+``-e `` with an environment name from ``tox.ini``. For example, to run
+the tests with Python 3.6 use::
+
+ tox -e py36
+
+You can also specify a comma-separated list of environmets, and use :ref:`tox’s
+parallel mode ` to run the tests on multiple environments in
+parallel::
+
+ tox -e py27,py36 -p auto
+
+To pass command-line options to :doc:`pytest `, add them after
+``--`` in your call to :doc:`tox `. Using ``--`` overrides the
+default positional arguments defined in ``tox.ini``, so you must include those
+default positional arguments (``scrapy tests``) after ``--`` as well::
+
+ tox -- scrapy tests -x # stop after first failure
+
+You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
+the Python 3.6 :doc:`tox ` environment using all your CPU cores::
+
+ tox -e py36 -- scrapy tests -n auto
+
+To see coverage report install :doc:`coverage `
+(``pip install coverage``) and run:
``coverage report``
see output of ``coverage --help`` for more options like html or xml report.
-.. _coverage: https://pypi.python.org/pypi/coverage
-
Writing tests
-------------
@@ -233,9 +270,9 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
-.. _pull request: https://help.github.com/send-pull-requests/
-.. _tox: https://pypi.python.org/pypi/tox
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
diff --git a/docs/faq.rst b/docs/faq.rst
index 8de680816..ee1a75cdd 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 3.5+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 2.6 support was dropped starting at Scrapy 0.20.
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
+Python 2 support was dropped in Scrapy 2.0.
.. note::
For Python 3 support on Windows, it is recommended to use
@@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
basically means that it crawls in `DFO order`_. This order is more convenient
-in most cases. If you do want to crawl in true `BFO order`_, you can do it by
+in most cases.
+
+If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
+While pending requests are below the configured values of
+:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
+concurrently. As a result, the first few requests of a crawl rarely follow the
+desired order. Lowering those settings to ``1`` enforces the desired order, but
+it significantly slows down the crawl as a whole.
+
+
My Scrapy crawler has memory leaks. What can I do?
--------------------------------------------------
@@ -319,6 +329,32 @@ I'm scraping a XML document and my XPath selector doesn't return any items
You may need to remove namespaces. See :ref:`removing-namespaces`.
+
+.. _faq-split-item:
+
+How to split an item into multiple items in an item pipeline?
+-------------------------------------------------------------
+
+:ref:`Item pipelines ` cannot yield multiple items per
+input item. :ref:`Create a spider middleware `
+instead, and use its
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+method for this puspose. For example::
+
+ from copy import deepcopy
+
+ from scrapy.item import BaseItem
+
+
+ class MultiplyItemsMiddleware:
+
+ def process_spider_output(self, response, result, spider):
+ for item in result:
+ if isinstance(item, (BaseItem, dict)):
+ for _ in range(item['multiply_by']):
+ yield deepcopy(item)
+
+
Running ``runspider`` I get ``error: No spider found in file: ``
--------------------------------------------------------------------------
@@ -327,9 +363,9 @@ conflicts with the name of one of the `Python standard library modules`_, such
as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
See :issue:`2680`.
+
.. _Python standard library modules: https://docs.python.org/py-modindex.html
.. _Python package: https://pypi.org/
-
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/index.rst b/docs/index.rst
index 0a96aa88e..6d5f9e77d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -4,7 +4,13 @@
Scrapy |version| documentation
==============================
-This documentation contains everything you need to know about Scrapy.
+Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used
+to crawl websites and extract structured data from their pages. It can be used
+for a wide range of purposes, from data mining to monitoring and automated
+testing.
+
+.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
+.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
Getting help
============
@@ -152,6 +158,7 @@ Solving specific problems
topics/practices
topics/broad-crawls
topics/developer-tools
+ topics/dynamic-content
topics/leaks
topics/media-pipeline
topics/deploy
@@ -177,6 +184,9 @@ Solving specific problems
:doc:`topics/developer-tools`
Learn how to scrape with your browser's developer tools.
+:doc:`topics/dynamic-content`
+ Read webpage data that is loaded dynamically.
+
:doc:`topics/leaks`
Learn how to find and get rid of memory leaks in your crawler.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index daec7fcb7..e924b5303 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
Installing Scrapy
=================
-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 3.5 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
@@ -102,10 +102,8 @@ just like any other Python package.
(See :ref:`platform-specific guides `
below for non-Python dependencies that you may need to install beforehand).
-Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
-
-* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
-* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
+Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
+only supports Python 3, make sure you created a Python 3 virtualenv.
.. _virtualenv: https://virtualenv.pypa.io
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
@@ -149,16 +147,12 @@ typically too old and slow to catch up with latest Scrapy.
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
these dependencies::
- sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
+ sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
-- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
+- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
are required for ``lxml``
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
-If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
-
- sudo apt-get install python3 python3-dev
-
Inside a :ref:`virtualenv `,
you can install Scrapy with ``pip`` after that::
@@ -290,5 +284,5 @@ For details, see `Issue #2473 `_.
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
-.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
+.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _conda-forge: https://conda-forge.org/
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8b2fef065..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').get(),
'author': quote.xpath('span/small/text()').get(),
+ 'text': quote.css('span.text::text').get(),
}
next_page = response.css('li.next a::attr("href")').get()
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 41e61542a..6b15a5fbd 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -78,9 +78,9 @@ Our first Spider
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
-:class:`scrapy.Spider` and define the initial requests to make, optionally how
-to follow links in the pages, and how to parse the downloaded page content to
-extract data.
+:class:`~scrapy.spiders.Spider` and define the initial requests to make,
+optionally how to follow links in the pages, and how to parse the downloaded
+page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@@ -205,7 +205,7 @@ Extracting data
---------------
The best way to learn how to extract data with Scrapy is trying selectors
-using the shell :ref:`Scrapy shell `. Run::
+using the :ref:`Scrapy shell `. Run::
scrapy shell 'http://quotes.toscrape.com/page/1/'
@@ -235,13 +235,16 @@ You will see something like::
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
- >>>
Using the shell, you can try selecting elements using `CSS`_ with the response
-object::
+object:
- >>> response.css('title')
- []
+.. invisible-code-block: python
+
+ response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+
+>>> response.css('title')
+[]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -296,8 +299,8 @@ expressions`_::
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools to inspect the HTML and come up
-with a selector (see section about :ref:`topics-developer-tools`).
+You can use your browser's developer tools to inspect the HTML and come up
+with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
@@ -372,6 +375,9 @@ we want::
We get a list of selectors for the quote HTML elements with::
>>> response.css("div.quote")
+ [,
+ ,
+ ...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
@@ -379,11 +385,11 @@ variable, so that we can run our CSS selectors directly on a particular quote::
>>> quote = response.css("div.quote")[0]
-Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
+Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
using the ``quote`` object we just created::
- >>> title = quote.css("span.text::text").get()
- >>> title
+ >>> text = quote.css("span.text::text").get()
+ >>> text
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
>>> author = quote.css("small.author::text").get()
>>> author
@@ -396,6 +402,12 @@ to get all of them::
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
+.. invisible-code-block: python
+
+ from sys import version_info
+
+.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
+
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary::
@@ -404,10 +416,9 @@ quotes elements and put them together into a Python dictionary::
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
- {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
- {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
- ... a few more of these, omitted for brevity
- >>>
+ {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+ {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+ ...
Extracting data in our spider
-----------------------------
@@ -511,7 +522,7 @@ We can try extracting it in the shell::
'Next →'
This gets the anchor element, but we want the attribute ``href``. For that,
-Scrapy supports a CSS extension that let's you select the attribute contents,
+Scrapy supports a CSS extension that lets you select the attribute contents,
like this::
>>> response.css('li.next a::attr(href)').get()
@@ -521,7 +532,7 @@ There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more)::
>>> response.css('li.next a').attrib['href']
- '/page/2'
+ '/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
diff --git a/docs/news.rst b/docs/news.rst
index 668473887..9dfd28508 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,570 @@
Release notes
=============
+.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
+ planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
+
+.. _release-1.8.0:
+
+Scrapy 1.8.0 (2019-10-28)
+-------------------------
+
+Highlights:
+
+* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
+ support official
+* New :meth:`Request.from_curl ` class method
+* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
+* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
+ :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Python 3.4 is no longer supported, and some of the minimum requirements of
+ Scrapy have also changed:
+
+ * cssselect_ 0.9.1
+ * cryptography_ 2.0
+ * lxml_ 3.5.0
+ * pyOpenSSL_ 16.2.0
+ * queuelib_ 1.4.2
+ * service_identity_ 16.0.0
+ * six_ 1.10.0
+ * Twisted_ 17.9.0 (16.0.0 with Python 2)
+ * zope.interface_ 4.1.3
+
+ (:issue:`3892`)
+
+* ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
+ consistency with similar classes (:issue:`3929`, :issue:`3982`)
+
+* If you are using a custom context factory
+ (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
+ accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
+ (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
+
+* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
+ into lists::
+
+ >>> item = MyItem()
+ >>> item['field'] = 'value1'
+ >>> loader = ItemLoader(item=item)
+ >>> item['field']
+ ['value1']
+
+ This is needed to allow adding values to existing fields
+ (``loader.add_value('field', 'value2')``).
+
+ (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
+ :issue:`4036`)
+
+See also :ref:`1.8-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+* A new :meth:`Request.from_curl ` class
+ method allows :ref:`creating a request from a cURL command
+ ` (:issue:`2985`, :issue:`3862`)
+
+* A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
+ parser to use. It includes built-in support for
+ :ref:`RobotFileParser `,
+ :ref:`Protego ` (default), :ref:`Reppy `, and
+ :ref:`Robotexclusionrulesparser `, and allows you to
+ :ref:`implement support for additional parsers
+ ` (:issue:`754`, :issue:`2669`,
+ :issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
+
+* A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
+ user agent string to use for robots.txt_ parsing (:issue:`3931`,
+ :issue:`3966`)
+
+* :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
+ ` parameter
+ (:issue:`781`, :issue:`4016`)
+
+* Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
+ the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
+ :issue:`3442`)
+
+* Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
+ ``True`` to enable debug-level messages about TLS connection parameters
+ after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
+
+* Callbacks that receive keyword arguments
+ (see :attr:`Request.cb_kwargs `) can now be
+ tested using the new :class:`@cb_kwargs
+ `
+ :ref:`spider contract ` (:issue:`3985`, :issue:`3988`)
+
+* When a :class:`@scrapes ` spider
+ contract fails, all missing fields are now reported (:issue:`766`,
+ :issue:`3939`)
+
+* :ref:`Custom log formats ` can now drop messages by
+ having the corresponding methods of the configured :setting:`LOG_FORMATTER`
+ return ``None`` (:issue:`3984`, :issue:`3987`)
+
+* A much improved completion definition is now available for Zsh_
+ (:issue:`4069`)
+
+
+Bug fixes
+~~~~~~~~~
+
+* :meth:`ItemLoader.load_item() ` no
+ longer makes later calls to :meth:`ItemLoader.get_output_value()
+ ` or
+ :meth:`ItemLoader.load_item() ` return
+ empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
+ :issue:`3998`, :issue:`4036`)
+
+* Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
+ :exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
+
+* :meth:`FilesPipeline.file_path
+ ` and
+ :meth:`ImagesPipeline.file_path
+ ` no longer choose
+ file extensions that are not `registered with IANA`_ (:issue:`1287`,
+ :issue:`3953`, :issue:`3954`)
+
+* When using botocore_ to persist files in S3, all botocore-supported headers
+ are properly mapped now (:issue:`3904`, :issue:`3905`)
+
+* FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
+ are now properly decoded (:issue:`3941`)
+
+* A memory-handling and error-handling issue in
+ :func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+* The documentation now covers how to define and configure a :ref:`custom log
+ format ` (:issue:`3616`, :issue:`3660`)
+
+* API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
+ and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
+
+* API documentation added for :class:`~scrapy.item.BaseItem` and
+ :class:`~scrapy.item.ItemMeta` (:issue:`3999`)
+
+* Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
+ :issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
+ :issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
+ :issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
+ :issue:`4081`, :issue:`4089`, :issue:`4093`)
+
+
+.. _1.8-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+* ``scrapy.xlib`` has been removed (:issue:`4015`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+* The LevelDB_ storage backend
+ (``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
+ :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
+ deprecated (:issue:`4085`, :issue:`4092`)
+
+* Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
+ variable is deprecated (:issue:`3910`)
+
+* ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
+ instead (:issue:`3999`)
+
+
+Other changes
+~~~~~~~~~~~~~
+
+* Minimum versions of optional Scrapy requirements that are covered by
+ continuous integration tests have been updated:
+
+ * botocore_ 1.3.23
+ * Pillow_ 3.4.2
+
+ Lower versions of these optional requirements may work, but it is not
+ guaranteed (:issue:`3892`)
+
+* GitHub templates for bug reports and feature requests (:issue:`3126`,
+ :issue:`3471`, :issue:`3749`, :issue:`3754`)
+
+* Continuous integration fixes (:issue:`3923`)
+
+* Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
+ :issue:`4023`, :issue:`4031`)
+
+
+.. _release-1.7.4:
+
+Scrapy 1.7.4 (2019-10-21)
+-------------------------
+
+Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
+side effects (:issue:`3897`, :issue:`3976`).
+
+As a result, when an item loader is initialized with an item,
+:meth:`ItemLoader.load_item() ` once again
+makes later calls to :meth:`ItemLoader.get_output_value()
+` or :meth:`ItemLoader.load_item()
+` return empty data.
+
+
+.. _release-1.7.3:
+
+Scrapy 1.7.3 (2019-08-01)
+-------------------------
+
+Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
+
+
+.. _release-1.7.2:
+
+Scrapy 1.7.2 (2019-07-23)
+-------------------------
+
+Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
+
+
+.. _release-1.7.1:
+
+Scrapy 1.7.1 (2019-07-18)
+-------------------------
+
+Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
+
+
+.. _release-1.7.0:
+
+Scrapy 1.7.0 (2019-07-18)
+-------------------------
+
+.. note:: Make sure you install Scrapy 1.7.1. The Scrapy 1.7.0 package in PyPI
+ is the result of an erroneous commit tagging and does not include all
+ the changes described below.
+
+Highlights:
+
+* Improvements for crawls targeting multiple domains
+* A cleaner way to pass arguments to callbacks
+* A new class for JSON requests
+* Improvements for rule-based spiders
+* New features for feed exports
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``429`` is now part of the :setting:`RETRY_HTTP_CODES` setting by default
+
+ This change is **backward incompatible**. If you don’t want to retry
+ ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
+
+* :class:`~scrapy.crawler.Crawler`,
+ :class:`CrawlerRunner.crawl ` and
+ :class:`CrawlerRunner.create_crawler `
+ no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
+ only accept a :class:`~scrapy.spiders.Spider` subclass now.
+
+ :class:`~scrapy.spiders.Spider` subclass instances were never meant to
+ work, and they were not working as one would expect: instead of using the
+ passed :class:`~scrapy.spiders.Spider` subclass instance, their
+ :class:`~scrapy.spiders.Spider.from_crawler` method was called to generate
+ a new instance.
+
+* Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting
+ may stop working. Scheduler priority queue classes now need to handle
+ :class:`~scrapy.http.Request` objects instead of arbitrary Python data
+ structures.
+
+See also :ref:`1.7-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+* A new scheduler priority queue,
+ ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
+ :ref:`enabled ` for a significant
+ scheduling improvement on crawls targetting multiple web domains, at the
+ cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+
+* A new :attr:`Request.cb_kwargs ` attribute
+ provides a cleaner way to pass keyword arguments to callback methods
+ (:issue:`1138`, :issue:`3563`)
+
+* A new :class:`JSONRequest ` class offers a more
+ convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
+
+* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
+ ``__init__`` method now receives the :class:`~scrapy.http.Response` object that
+ originated the request as its second argument (:issue:`3682`)
+
+* A new ``restrict_text`` parameter for the
+ :attr:`LinkExtractor `
+ ``__init__`` method allows filtering links by linking text (:issue:`3622`,
+ :issue:`3635`)
+
+* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
+ for feeds exported to Amazon S3 (:issue:`3607`)
+
+* A new :setting:`FEED_STORAGE_FTP_ACTIVE` setting allows using FTP’s active
+ connection mode for feeds exported to FTP servers (:issue:`3829`)
+
+* A new :setting:`METAREFRESH_IGNORE_TAGS` setting allows overriding which
+ HTML tags are ignored when searching a response for HTML meta tags that
+ trigger a redirect (:issue:`1422`, :issue:`3768`)
+
+* A new :reqmeta:`redirect_reasons` request meta key exposes the reason
+ (status code, meta refresh) behind every followed redirect (:issue:`3581`,
+ :issue:`3687`)
+
+* The ``SCRAPY_CHECK`` variable is now set to the ``true`` string during runs
+ of the :command:`check` command, which allows :ref:`detecting contract
+ check runs from code ` (:issue:`3704`,
+ :issue:`3739`)
+
+* A new :meth:`Item.deepcopy() ` method makes it
+ easier to :ref:`deep-copy items ` (:issue:`1493`,
+ :issue:`3671`)
+
+* :class:`~scrapy.extensions.corestats.CoreStats` also logs
+ ``elapsed_time_seconds`` now (:issue:`3638`)
+
+* Exceptions from :class:`~scrapy.loader.ItemLoader` :ref:`input and output
+ processors ` are now more verbose
+ (:issue:`3836`, :issue:`3840`)
+
+* :class:`~scrapy.crawler.Crawler`,
+ :class:`CrawlerRunner.crawl ` and
+ :class:`CrawlerRunner.create_crawler `
+ now fail gracefully if they receive a :class:`~scrapy.spiders.Spider`
+ subclass instance instead of the subclass itself (:issue:`2283`,
+ :issue:`3610`, :issue:`3872`)
+
+
+Bug fixes
+~~~~~~~~~
+
+* :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+ is now also invoked for generators (:issue:`220`, :issue:`2061`)
+
+* System exceptions like KeyboardInterrupt_ are no longer caught
+ (:issue:`3726`)
+
+* :meth:`ItemLoader.load_item() ` no
+ longer makes later calls to :meth:`ItemLoader.get_output_value()
+ ` or
+ :meth:`ItemLoader.load_item() ` return
+ empty data (:issue:`3804`, :issue:`3819`)
+
+* The images pipeline (:class:`~scrapy.pipelines.images.ImagesPipeline`) no
+ longer ignores these Amazon S3 settings: :setting:`AWS_ENDPOINT_URL`,
+ :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
+ (:issue:`3625`)
+
+* Fixed a memory leak in ``scrapy.pipelines.media.MediaPipeline`` affecting,
+ for example, non-200 responses and exceptions from custom middlewares
+ (:issue:`3813`)
+
+* Requests with private callbacks are now correctly unserialized from disk
+ (:issue:`3790`)
+
+* :meth:`FormRequest.from_response() `
+ now handles invalid methods like major web browsers (:issue:`3777`,
+ :issue:`3794`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+* A new topic, :ref:`topics-dynamic-content`, covers recommended approaches
+ to read dynamically-loaded data (:issue:`3703`)
+
+* :ref:`topics-broad-crawls` now features information about memory usage
+ (:issue:`1264`, :issue:`3866`)
+
+* The documentation of :class:`~scrapy.spiders.Rule` now covers how to access
+ the text of a link when using :class:`~scrapy.spiders.CrawlSpider`
+ (:issue:`3711`, :issue:`3712`)
+
+* A new section, :ref:`httpcache-storage-custom`, covers writing a custom
+ cache storage backend for
+ :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
+ (:issue:`3683`, :issue:`3692`)
+
+* A new :ref:`FAQ ` entry, :ref:`faq-split-item`, explains what to do
+ when you want to split an item into multiple items from an item pipeline
+ (:issue:`2240`, :issue:`3672`)
+
+* Updated the :ref:`FAQ entry about crawl order ` to explain why
+ the first few requests rarely follow the desired order (:issue:`1739`,
+ :issue:`3621`)
+
+* The :setting:`LOGSTATS_INTERVAL` setting (:issue:`3730`), the
+ :meth:`FilesPipeline.file_path `
+ and
+ :meth:`ImagesPipeline.file_path `
+ methods (:issue:`2253`, :issue:`3609`) and the
+ :meth:`Crawler.stop() ` method (:issue:`3842`)
+ are now documented
+
+* Some parts of the documentation that were confusing or misleading are now
+ clearer (:issue:`1347`, :issue:`1789`, :issue:`2289`, :issue:`3069`,
+ :issue:`3615`, :issue:`3626`, :issue:`3668`, :issue:`3670`, :issue:`3673`,
+ :issue:`3728`, :issue:`3762`, :issue:`3861`, :issue:`3882`)
+
+* Minor documentation fixes (:issue:`3648`, :issue:`3649`, :issue:`3662`,
+ :issue:`3674`, :issue:`3676`, :issue:`3694`, :issue:`3724`, :issue:`3764`,
+ :issue:`3767`, :issue:`3791`, :issue:`3797`, :issue:`3806`, :issue:`3812`)
+
+.. _1.7-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+The following deprecated APIs have been removed (:issue:`3578`):
+
+* ``scrapy.conf`` (use :attr:`Crawler.settings
+ `)
+
+* From ``scrapy.core.downloader.handlers``:
+
+ * ``http.HttpDownloadHandler`` (use ``http10.HTTP10DownloadHandler``)
+
+* ``scrapy.loader.ItemLoader._get_values`` (use ``_get_xpathvalues``)
+
+* ``scrapy.loader.XPathItemLoader`` (use :class:`~scrapy.loader.ItemLoader`)
+
+* ``scrapy.log`` (see :ref:`topics-logging`)
+
+* From ``scrapy.pipelines``:
+
+ * ``files.FilesPipeline.file_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.file_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.image_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.thumb_key`` (use ``thumb_path``)
+
+* From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``:
+
+ * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`)
+
+* From ``scrapy.selector.csstranslator``:
+
+ * ``ScrapyGenericTranslator`` (use parsel.csstranslator.GenericTranslator_)
+
+ * ``ScrapyHTMLTranslator`` (use parsel.csstranslator.HTMLTranslator_)
+
+ * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_)
+
+* From :class:`~scrapy.selector.Selector`:
+
+ * ``_root`` (both the ``__init__`` method argument and the object property, use
+ ``root``)
+
+ * ``extract_unquoted`` (use ``getall``)
+
+ * ``select`` (use ``xpath``)
+
+* From :class:`~scrapy.selector.SelectorList`:
+
+ * ``extract_unquoted`` (use ``getall``)
+
+ * ``select`` (use ``xpath``)
+
+ * ``x`` (use ``xpath``)
+
+* ``scrapy.spiders.BaseSpider`` (use :class:`~scrapy.spiders.Spider`)
+
+* From :class:`~scrapy.spiders.Spider` (and subclasses):
+
+ * ``DOWNLOAD_DELAY`` (use :ref:`download_delay
+ `)
+
+ * ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
+
+* ``scrapy.spiders.spiders`` (use :class:`~scrapy.spiderloader.SpiderLoader`)
+
+* ``scrapy.telnet`` (use :mod:`scrapy.extensions.telnet`)
+
+* From ``scrapy.utils.python``:
+
+ * ``str_to_unicode`` (use ``to_unicode``)
+
+ * ``unicode_to_str`` (use ``to_bytes``)
+
+* ``scrapy.utils.response.body_or_str``
+
+The following deprecated settings have also been removed (:issue:`3578`):
+
+* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+* The ``queuelib.PriorityQueue`` value for the
+ :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
+ ``scrapy.pqueues.ScrapyPriorityQueue`` instead.
+
+* ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
+ do not accept two arguments are deprecated.
+
+* The following modules are deprecated:
+
+ * ``scrapy.utils.http`` (use `w3lib.http`_)
+
+ * ``scrapy.utils.markup`` (use `w3lib.html`_)
+
+ * ``scrapy.utils.multipart`` (use `urllib3`_)
+
+* The ``scrapy.utils.datatypes.MergeDict`` class is deprecated for Python 3
+ code bases. Use :class:`~collections.ChainMap` instead. (:issue:`3878`)
+
+* The ``scrapy.utils.gz.is_gzipped`` function is deprecated. Use
+ ``scrapy.utils.gz.gzip_magic_number`` instead.
+
+.. _urllib3: https://urllib3.readthedocs.io/en/latest/index.html
+.. _w3lib.html: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.html
+.. _w3lib.http: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.http
+
+
+Other changes
+~~~~~~~~~~~~~
+
+* It is now possible to run all tests from the same tox_ environment in
+ parallel; the documentation now covers :ref:`this and other ways to run
+ tests ` (:issue:`3707`)
+
+* It is now possible to generate an API documentation coverage report
+ (:issue:`3806`, :issue:`3810`, :issue:`3860`)
+
+* The :ref:`documentation policies ` now require
+ docstrings_ (:issue:`3701`) that follow `PEP 257`_ (:issue:`3748`)
+
+* Internal fixes and cleanup (:issue:`3629`, :issue:`3643`, :issue:`3684`,
+ :issue:`3698`, :issue:`3734`, :issue:`3735`, :issue:`3736`, :issue:`3737`,
+ :issue:`3809`, :issue:`3821`, :issue:`3825`, :issue:`3827`, :issue:`3833`,
+ :issue:`3857`, :issue:`3877`)
+
.. _release-1.6.0:
Scrapy 1.6.0 (2019-01-30)
@@ -53,7 +617,7 @@ case of Scrapy spiders: callbacks are usually called several times,
on different pages.
If you're using custom ``Selector`` or ``SelectorList`` subclasses,
-a **backwards incompatible** change in parsel may affect your code.
+a **backward incompatible** change in parsel may affect your code.
See `parsel changelog`_ for a detailed description, as well as for the
full list of improvements.
@@ -62,7 +626,7 @@ full list of improvements.
Telnet console
~~~~~~~~~~~~~~
-**Backwards incompatible**: Scrapy's telnet console now requires username
+**Backward incompatible**: Scrapy's telnet console now requires username
and password. See :ref:`topics-telnetconsole` for more details. This change
fixes a **security issue**; see :ref:`release-1.5.2` release notes for details.
@@ -149,7 +713,7 @@ Documentation improvements
* improved links to beginner resources in the tutorial
(:issue:`3367`, :issue:`3468`);
* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`);
-* remove unused `DEPTH_STATS` option from docs (:issue:`3245`);
+* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`);
* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`,
:issue:`3605`).
@@ -209,14 +773,14 @@ Scrapy 1.5.2 (2019-01-22)
exploit it from Scrapy, but it is very easy to trick a browser to do so and
elevates the risk for local development environment.
- *The fix is backwards incompatible*, it enables telnet user-password
+ *The fix is backward incompatible*, it enables telnet user-password
authentication by default with a random generated password. If you can't
- upgrade right away, please consider setting :setting:`TELNET_CONSOLE_PORT`
+ upgrade right away, please consider setting :setting:`TELNETCONSOLE_PORT`
out of its default value.
See :ref:`telnet console ` documentation for more info
-* Backport CI build failure under GCE environemnt due to boto import error.
+* Backport CI build failure under GCE environment due to boto import error.
.. _release-1.5.1:
@@ -256,15 +820,15 @@ Some highlights:
* Better default handling of HTTP 308, 522 and 524 status codes.
* Documentation is improved, as usual.
-Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Scrapy 1.5 drops support for Python 3.3.
* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
- **This is technically backwards-incompatible**; override
+ **This is technically backward-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
* Logging of settings overridden by ``custom_settings`` is fixed;
- **this is technically backwards-incompatible** because the logger
+ **this is technically backward-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
parsing Scrapy logs, please update your log parsers (:issue:`1343`).
* LinkExtractor now ignores ``m4v`` extension by default, this is change
@@ -301,11 +865,11 @@ Bug fixes
~~~~~~~~~
- Fix logging of settings overridden by ``custom_settings``;
- **this is technically backwards-incompatible** because the logger
+ **this is technically backward-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
update your log parsers if needed (:issue:`1343`)
- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
- **This is technically backwards-incompatible**; override
+ **This is technically backward-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
- Fix PyPy and PyPy3 test failures, support them officially
(:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
@@ -415,18 +979,20 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
Enjoy! (Or read on for the rest of changes in this release.)
-Deprecations and Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Deprecations and Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
+- Default to ``canonicalize=False`` in
+ :class:`scrapy.linkextractors.LinkExtractor
+ `
(:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
- **warning, this is technically backwards-incompatible**
+ **warning, this is technically backward-incompatible**
- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
- **this is technically backwards-incompatible** so please check if you have
+ **this is technically backward-incompatible** so please check if you have
any non-default ``MEMUSAGE_***`` options set.
- ``EDITOR`` environment variable now takes precedence over ``EDITOR``
option defined in settings.py (:issue:`1829`); Scrapy default settings
- no longer depend on environment variables. **This is technically a backwards
+ no longer depend on environment variables. **This is technically a backward
incompatible change**.
- ``Spider.make_requests_from_url`` is deprecated
(:issue:`1728`, fixes :issue:`1495`).
@@ -454,10 +1020,13 @@ New Features
- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
-- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
-- :ref:`Media downloads `, with :class:`FilesPipelines`
- or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
- using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
+- ``scrapy.utils.datatypes.CaselessDict`` now accepts ``Mapping`` instances and
+ not only dicts (:issue:`2646`)
+- :ref:`Media downloads `, with
+ :class:`~scrapy.pipelines.files.FilesPipeline` or
+ :class:`~scrapy.pipelines.images.ImagesPipeline`, can now optionally handle
+ HTTP redirects using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting
+ (:issue:`2616`, fixes :issue:`2004`)
- Accept non-complete responses from websites using a new
:setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
- Optional pretty-printing of JSON and XML items via
@@ -477,8 +1046,8 @@ Bug fixes
- LinkExtractor now strips leading and trailing whitespaces from attributes
(:issue:`2547`, fixes :issue:`1614`)
-- Properly handle whitespaces in action attribute in :class:`FormRequest`
- (:issue:`2548`)
+- Properly handle whitespaces in action attribute in
+ :class:`~scrapy.http.FormRequest` (:issue:`2548`)
- Buffer CONNECT response bytes from proxy until all HTTP headers are received
(:issue:`2495`, fixes :issue:`2491`)
- FTP downloader now works on Python 3, provided you use Twisted>=17.1
@@ -511,7 +1080,8 @@ Cleanups & Refactoring
fixes :issue:`2560`)
- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
-- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
+- Catch more specific ``os.error`` exception in
+ ``scrapy.pipelines.files.FSFilesStore`` (:issue:`2644`)
- Change "localhost" test server certificate (:issue:`2720`)
- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
@@ -528,7 +1098,8 @@ Documentation
(:issue:`2477`, fixes :issue:`2475`)
- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
- Rearrange selector sections (:issue:`2705`)
-- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
+- Remove ``__nonzero__`` from :class:`~scrapy.selector.SelectorList`
+ docs (:issue:`2683`)
- Mention how to disable request filtering in documentation of
:setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
@@ -636,10 +1207,10 @@ New Features
scrapy shell now follow HTTP redirections by default (:issue:`2290`);
See :command:`fetch` and :command:`shell` for details.
- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
- this is technically **backwards incompatible** so please check your log parsers.
+ this is technically **backward incompatible** so please check your log parsers.
- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
- this is **backwards incompatible** if you have log parsers expecting the short
+ this is **backward incompatible** if you have log parsers expecting the short
logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
set to ``True``.
@@ -750,11 +1321,11 @@ Bug fixes
~~~~~~~~~
- DefaultRequestHeaders middleware now runs before UserAgent middleware
- (:issue:`2088`). **Warning: this is technically backwards incompatible**,
+ (:issue:`2088`). **Warning: this is technically backward incompatible**,
though we consider this a bug fix.
- HTTP cache extension and plugins that use the ``.scrapy`` data directory now
work outside projects (:issue:`1581`). **Warning: this is technically
- backwards incompatible**, though we consider this a bug fix.
+ backward incompatible**, though we consider this a bug fix.
- ``Selector`` does not allow passing both ``response`` and ``text`` anymore
(:issue:`2153`).
- Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).
@@ -929,18 +1500,18 @@ This 1.1 release brings a lot of interesting features and bug fixes:
this behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file
after creating a new project.
- Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
- If you use ``PythonItemExporter``, you may want to update your code to
- disable binary mode which is now deprecated.
+ If you use :class:`~scrapy.exporters.PythonItemExporter`, you may want to
+ update your code to disable binary mode which is now deprecated.
- Accept XML node names containing dots as valid (:issue:`1533`).
- When uploading files or images to S3 (with ``FilesPipeline`` or
``ImagesPipeline``), the default ACL policy is now "private" instead
- of "public" **Warning: backwards incompatible!**.
+ of "public" **Warning: backward incompatible!**.
You can use :setting:`FILES_STORE_S3_ACL` to change it.
- We've reimplemented ``canonicalize_url()`` for more correct output,
especially for URLs with non-ASCII characters (:issue:`1947`).
This could change link extractors output compared to previous scrapy versions.
This may also invalidate some cache entries you could still have from pre-1.1 runs.
- **Warning: backwards incompatible!**.
+ **Warning: backward incompatible!**.
Keep reading for more details on other improvements and bug fixes.
@@ -973,7 +1544,7 @@ Additional New Features and Enhancements
- Support for bpython and configure preferred Python shell via
``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`).
- Support URLs without scheme (:issue:`1498`)
- **Warning: backwards incompatible!**
+ **Warning: backward incompatible!**
- Bring back support for relative file path (:issue:`1710`, :issue:`1550`).
- Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check
@@ -1056,7 +1627,7 @@ Bugfixes
~~~~~~~~
- Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
- response anymore (:issue:`1289`). **Warning: backwards incompatible!**
+ response anymore (:issue:`1289`). **Warning: backward incompatible!**
- Support empty password for http_proxy config (:issue:`1274`).
- Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`).
- Support link rel attribute with multiple values (:issue:`1201`).
@@ -1068,8 +1639,8 @@ Bugfixes
- Fixed bug on ``XMLItemExporter`` with non-string fields in
items (:issue:`1738`).
- Fixed startproject command in OS X (:issue:`1635`).
-- Fixed PythonItemExporter and CSVExporter for non-string item
- types (:issue:`1737`).
+- Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for
+ non-string item types (:issue:`1737`).
- Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
:issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
- Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
@@ -1313,7 +1884,7 @@ Module Relocations
There’s been a large rearrangement of modules trying to improve the general
structure of Scrapy. Main changes were separating various subpackages into
-new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp`
+new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp``
into top level packages. Backward compatibility was kept among internal
relocations, while importing deprecated modules expect warnings indicating
their new place.
@@ -1344,7 +1915,7 @@ Outsourced packages
| | /scrapy-plugins/scrapy-jsonrpc>`_ |
+-------------------------------------+-------------------------------------+
-`scrapy.contrib_exp` and `scrapy.contrib` dissolutions
+``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions
+-------------------------------------+-------------------------------------+
| Old location | New location |
@@ -1556,7 +2127,7 @@ Code refactoring
(:issue:`1078`)
- Pydispatch pep8 (:issue:`992`)
- Removed unused 'load=False' parameter from walk_modules() (:issue:`871`)
-- For consistency, use `job_dir` helper in `SpiderState` extension.
+- For consistency, use ``job_dir`` helper in ``SpiderState`` extension.
(:issue:`805`)
- rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`)
@@ -1646,7 +2217,7 @@ Scrapy 0.24.2 (2014-07-08)
Scrapy 0.24.1 (2014-06-27)
--------------------------
-- Fix deprecated CrawlerSettings and increase backwards compatibility with
+- Fix deprecated CrawlerSettings and increase backward compatibility with
.defaults attribute (:commit:`8e3f20a`)
@@ -1669,10 +2240,10 @@ Enhancements
cache middleware (:issue:`541`, :issue:`500`, :issue:`571`)
- Expose current crawler in Scrapy shell (:issue:`557`)
- Improve testsuite comparing CSV and XML exporters (:issue:`570`)
-- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`)
+- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`)
- Support process_links as generator in CrawlSpider (:issue:`555`)
- Verbose logging and new stats counters for DupeFilter (:issue:`553`)
-- Add a mimetype parameter to `MailSender.send()` (:issue:`602`)
+- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`)
- Generalize file pipeline log messages (:issue:`622`)
- Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`)
- Converted SEP documents to rst format (:issue:`629`, :issue:`630`,
@@ -1691,20 +2262,20 @@ Enhancements
- Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`)
- Infer exporter's output format from filename extensions
(:issue:`546`, :issue:`659`, :issue:`760`)
-- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`)
+- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`)
- Remove pep8 warnings in project and spider templates (:issue:`698`)
-- Tests and docs for `request_fingerprint` function (:issue:`597`)
-- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`)
+- Tests and docs for ``request_fingerprint`` function (:issue:`597`)
+- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`)
- Set exit code to non-zero when contracts fails (:issue:`727`)
- Add a setting to control what class is instanciated as Downloader component
(:issue:`738`)
-- Pass response in `item_dropped` signal (:issue:`724`)
-- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`)
-- Document `spider.closed()` shortcut (:issue:`719`)
-- Document `request_scheduled` signal (:issue:`746`)
+- Pass response in ``item_dropped`` signal (:issue:`724`)
+- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`)
+- Document ``spider.closed()`` shortcut (:issue:`719`)
+- Document ``request_scheduled`` signal (:issue:`746`)
- Add a note about reporting security issues (:issue:`697`)
- Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
-- Sort spider list output of `scrapy list` command (:issue:`742`)
+- Sort spider list output of ``scrapy list`` command (:issue:`742`)
- Multiple documentation enhancemens and fixes
(:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
:issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,
@@ -1772,23 +2343,23 @@ Scrapy 0.22.0 (released 2014-01-17)
Enhancements
~~~~~~~~~~~~
-- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
- To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage`
+- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
+ To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
- Selectors register EXSLT namespaces by default (:issue:`472`)
- Unify item loaders similar to selectors renaming (:issue:`461`)
-- Make `RFPDupeFilter` class easily subclassable (:issue:`533`)
+- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`)
- Improve test coverage and forthcoming Python 3 support (:issue:`525`)
- Promote startup info on settings and middleware to INFO level (:issue:`520`)
-- Support partials in `get_func_args` util (:issue:`506`, issue:`504`)
+- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
- Allow running indiviual tests via tox (:issue:`503`)
- Update extensions ignored by link extractors (:issue:`498`)
- Add middleware methods to get files/images/thumbs paths (:issue:`490`)
- Improve offsite middleware tests (:issue:`478`)
- Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`)
-- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`)
+- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`)
- Support defining http error handling using settings (:issue:`466`)
- Use modern python idioms wherever you find legacies (:issue:`497`)
- Improve and correct documentation
@@ -1799,14 +2370,14 @@ Fixes
~~~~~
- Update Selector class imports in CrawlSpider template (:issue:`484`)
-- Fix unexistent reference to `engine.slots` (:issue:`464`)
-- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`)
+- Fix unexistent reference to ``engine.slots`` (:issue:`464`)
+- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`)
- Warn when subclassing XPathItemLoader, previously it only warned on
instantiation. (:issue:`523`)
- Warn when subclassing XPathSelector, previously it only warned on
instantiation. (:issue:`537`)
- Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`)
-- Fix overriding url in `FormRequest.from_response()` (:issue:`507`)
+- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`)
- Fix tests runner under pip 1.5 (:issue:`513`)
- Fix logging error when spider name is unicode (:issue:`479`)
@@ -1833,7 +2404,7 @@ Enhancements
(modifying them had been deprecated for a long time)
- :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list)
- Sitemap spider can fetch alternate URLs (:issue:`360`)
-- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`)
+- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`)
- Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`)
- New item exporter using native python types with nesting support (:issue:`366`)
- Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`)
@@ -1844,13 +2415,13 @@ Enhancements
- Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
- Remove multi spider support from multiple core components
(:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
-- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages.
+- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages.
- Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`)
- Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`)
- Use importlib for parametric imports (:issue:`445`)
- Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`)
- Bugfix crawling shutdown on SIGINT (:issue:`450`)
-- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`)
+- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`)
- Do not silence download errors when request errback raises an exception (:commit:`684cfc0`)
Bugfixes
@@ -1865,8 +2436,8 @@ Bugfixes
- Improve request-response docs (:issue:`391`)
- Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`)
- Improve django integration docs (:issue:`404`)
-- Document `bindaddress` request meta (:commit:`37c24e01d7`)
-- Improve `Request` class documentation (:issue:`226`)
+- Document ``bindaddress`` request meta (:commit:`37c24e01d7`)
+- Improve ``Request`` class documentation (:issue:`226`)
Other
~~~~~
@@ -1875,7 +2446,7 @@ Other
- Add `cssselect`_ python package as install dependency
- Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on.
- Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support.
-- Running test suite now requires `mock` python library (:issue:`390`)
+- Running test suite now requires ``mock`` python library (:issue:`390`)
Thanks
@@ -1929,7 +2500,7 @@ Scrapy 0.18.3 (released 2013-10-03)
Scrapy 0.18.2 (released 2013-09-03)
-----------------------------------
-- Backport `scrapy check` command fixes and backward compatible multi
+- Backport ``scrapy check`` command fixes and backward compatible multi
crawler process(:issue:`339`)
Scrapy 0.18.1 (released 2013-08-27)
@@ -1958,36 +2529,36 @@ Scrapy 0.18.0 (released 2013-08-09)
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
- Use lxml recover option to parse sitemaps (:issue:`347`)
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
-- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`)
-- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`)
-- Support `dont_cache` request meta flag (:issue:`19`)
-- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`)
-- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`)
-- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`)
+- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
+- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`)
+- Support ``dont_cache`` request meta flag (:issue:`19`)
+- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`)
+- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`)
+- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`)
- Support nested items in xml exporter (:issue:`66`)
- Improve cookies handling performance (:issue:`77`)
- Log dupe filtered requests once (:issue:`105`)
- Split redirection middleware into status and meta based middlewares (:issue:`78`)
- Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`)
-- Support xpath form selection on `FormRequest.from_response` (:issue:`185`)
-- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`)
+- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`)
+- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`)
- Bugfix signal dispatching on pypi interpreter (:issue:`205`)
- Improve request delay and concurrency handling (:issue:`206`)
-- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`)
+- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`)
- Allow customization of messages logged by engine (:issue:`214`)
-- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`)
+- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`)
- Extend Scrapy commands using setuptools entry points (:issue:`260`)
-- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`)
-- Support `settings.getdict` (:issue:`269`)
-- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`)
-- Added `Item.copy` (:issue:`290`)
+- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`)
+- Support ``settings.getdict`` (:issue:`269`)
+- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`)
+- Added ``Item.copy`` (:issue:`290`)
- Collect idle downloader slots (:issue:`297`)
-- Add `ftp://` scheme downloader handler (:issue:`329`)
+- Add ``ftp://`` scheme downloader handler (:issue:`329`)
- Added downloader benchmark webserver and spider tools :ref:`benchmarking`
- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
- Add scrapy commands using external libraries (:issue:`260`)
- Added ``--pdb`` option to ``scrapy`` command line tool
-- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
+- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
@@ -2092,7 +2663,7 @@ Scrapy 0.16.1 (released 2012-10-26)
-----------------------------------
- fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
-- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`)
+- better backward compatibility for scrapy.conf.settings (:commit:`3403089`)
- extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`)
- removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`)
- fix dashes under rst headers (:commit:`fa4f7f9`)
@@ -2107,13 +2678,13 @@ Scrapy changes:
- added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
- added options ``-o`` and ``-t`` to the :command:`runspider` command
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
-- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals.
-- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
+- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
+- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
- documented :ref:`topics-api`
-- `lxml` is now the default selectors backend instead of `libxml2`
+- ``lxml`` is now the default selectors backend instead of ``libxml2``
- ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_
- removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm``
- SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
@@ -2132,7 +2703,7 @@ Scrapy changes:
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
-- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
+- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled
@@ -2206,16 +2777,16 @@ New features and settings
- New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`)
- Add boto 2.0 support for S3 downloader handler (:rev:`2763`)
- Added `marshal`_ to formats supported by feed exports (:rev:`2744`)
-- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`)
+- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
- check the documentation for more details
- Added builtin caching DNS resolver (:rev:`2728`)
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
-- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`)
-- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`)
-- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
+- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`)
+- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`)
+- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
- Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`)
- Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`)
- Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`)
@@ -2223,19 +2794,19 @@ New features and settings
- Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`)
- Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`)
- Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`)
-- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`.
+- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``.
- Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`)
-- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`)
+- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`)
- Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines.
- Added :signal:`spider_error` signal (:rev:`2628`)
- Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`)
-- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
+- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
- Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`)
- Added new DBM HTTP cache storage backend (:rev:`2576`)
- Added ``listjobs.json`` API to Scrapyd (:rev:`2571`)
- ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`)
- Added namespace support to ``xmliter_lxml`` (:rev:`2552`)
-- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`)
+- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`)
- Several improvements to Scrapyd and Link extractors
Code rearranged and removed
@@ -2249,17 +2820,17 @@ Code rearranged and removed
- Reduced Scrapy codebase by striping part of Scrapy code into two new libraries:
- `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
- `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
-- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`)
-- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot
+- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`)
+- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
- Removed support for default field values in Scrapy items (:rev:`2616`)
- Removed experimental crawlspider v2 (:rev:`2632`)
-- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
+- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
- Removed deprecated Execution Queue (:rev:`2704`)
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
- removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`)
- Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`)
-- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept.
+- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept.
Scrapy 0.12
-----------
@@ -2269,7 +2840,8 @@ The numbers like #NNN reference tickets in the old issue tracker (Trac) which is
New features and improvements
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
+- Passed item is now sent in the ``item`` argument of the :signal:`item_passed
+ ` (#273)
- Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
- HTTP cache now stored by default in the project data dir (#279)
- Added project data storage directory (#276, #277)
@@ -2289,13 +2861,13 @@ Scrapyd changes
- Scrapyd now uses one process per spider
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
- A minimal web ui was added, available at http://localhost:6800 by default
-- There is now a `scrapy server` command to start a Scrapyd server of the current project
+- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
Changes to settings
~~~~~~~~~~~~~~~~~~~
-- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware
-- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire".
+- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware
+- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire".
Deprecated/obsoleted functionality
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2326,17 +2898,17 @@ New features and improvements
- Splitted Debian package into two packages - the library and the service (#187)
- Scrapy log refactoring (#188)
- New extension for keeping persistent spider contexts among different runs (#203)
-- Added `dont_redirect` request.meta key for avoiding redirects (#233)
-- Added `dont_retry` request.meta key for avoiding retries (#234)
+- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
+- Added ``dont_retry`` request.meta key for avoiding retries (#234)
Command-line tool changes
~~~~~~~~~~~~~~~~~~~~~~~~~
-- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199)
- - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project
- - Added `scrapy.bat` script for running more conveniently from Windows
+- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199)
+ - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project
+ - Added ``scrapy.bat`` script for running more conveniently from Windows
- Added bash completion to command-line tool (#210)
-- Renamed command `start` to `runserver` (#209)
+- Renamed command ``start`` to ``runserver`` (#209)
API changes
~~~~~~~~~~~
@@ -2345,7 +2917,7 @@ API changes
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
-- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
+- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
- Changes to Scrapy Manager (now called "Crawler"):
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
@@ -2356,11 +2928,11 @@ API changes
- ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector``
- default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201)
- changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- moved ``scrapy.core.signals`` module to ``scrapy.signals``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- added ``handles_request()`` class method to ``BaseSpider``
- dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead)
- dropped ``component`` argument of ``scrapy.log.msg()`` function
@@ -2431,8 +3003,8 @@ New features
- Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`)
- Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`)
-Backwards-incompatible changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`)
- Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`)
@@ -2470,13 +3042,35 @@ First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
+.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-.. _w3lib: https://github.com/scrapy/w3lib
-.. _scrapely: https://github.com/scrapy/scrapely
-.. _marshal: https://docs.python.org/2/library/marshal.html
-.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
-.. _lxml: http://lxml.de/
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
-.. _resource: https://docs.python.org/2/library/resource.html
+.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _cryptography: https://cryptography.io/en/latest/
+.. _cssselect: https://github.com/scrapy/cssselect/
+.. _docstrings: https://docs.python.org/glossary.html#term-docstring
+.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _LevelDB: https://github.com/google/leveldb
+.. _lxml: http://lxml.de/
+.. _marshal: https://docs.python.org/2/library/marshal.html
+.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
+.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
+.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _Pillow: https://python-pillow.org/
+.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
.. _queuelib: https://github.com/scrapy/queuelib
-.. _cssselect: https://github.com/SimonSapin/cssselect
+.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
+.. _resource: https://docs.python.org/2/library/resource.html
+.. _robots.txt: http://www.robotstxt.org/
+.. _scrapely: https://github.com/scrapy/scrapely
+.. _service_identity: https://service-identity.readthedocs.io/en/stable/
+.. _six: https://six.readthedocs.io/
+.. _tox: https://pypi.python.org/pypi/tox
+.. _Twisted: https://twistedmatrix.com/trac/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
+.. _w3lib: https://github.com/scrapy/w3lib
+.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
+.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
+.. _Zsh: https://www.zsh.org/
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8e7611d21..0ed11c4dc 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,5 @@
-Sphinx>=1.6
-sphinx_rtd_theme
\ No newline at end of file
+-r ../requirements-py3.txt
+Sphinx>=2.1
+sphinx-hoverxref
+sphinx-notfound-page
+sphinx_rtd_theme
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 985cc0433..1c461a511 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -94,11 +94,13 @@ how you :ref:`configure the downloader middlewares
.. method:: crawl(\*args, \**kwargs)
Starts the crawler by instantiating its spider class with the given
- `args` and `kwargs` arguments, while setting the execution engine in
+ ``args`` and ``kwargs`` arguments, while setting the execution engine in
motion.
Returns a deferred that is fired when the crawl is finished.
+ .. automethod:: stop
+
.. autoclass:: CrawlerRunner
:members:
@@ -154,7 +156,7 @@ Settings API
SpiderLoader API
================
-.. module:: scrapy.loader
+.. module:: scrapy.spiderloader
:synopsis: The spider loader
.. class:: SpiderLoader
@@ -180,7 +182,7 @@ SpiderLoader API
.. method:: load(spider_name)
Get the Spider class with the given name. It'll look into the previously
- loaded spiders for a spider class with name `spider_name` and will raise
+ loaded spiders for a spider class with name ``spider_name`` and will raise
a KeyError if not found.
:param spider_name: spider class name
@@ -271,5 +273,3 @@ class (which they all inherit from).
Close the given spider. After this is called, no more specific stats
can be accessed or collected.
-
-.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 4ac39ad2d..ae25dfa2f 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -166,11 +166,10 @@ for concurrency.
For more information about asynchronous programming and Twisted see these
links:
-* `Introduction to Deferreds in Twisted`_
+* :doc:`twisted:core/howto/defer-intro`
* `Twisted - hello, asynchronous programming`_
* `Twisted Introduction - Krondo`_
.. _Twisted: https://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index eb02086dc..1ab08d949 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -39,24 +39,50 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
concrete suggestions of Scrapy settings to tune in order to achieve an
efficient broad crawl.
+.. _broad-crawls-scheduler-priority-queue:
+
+Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
+=================================================
+
+Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
+It works best during single-domain crawl. It does not work well with crawling
+many different domains in parallel
+
+To apply the recommended priority queue use::
+
+ SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
+.. _broad-crawls-concurrency:
+
Increase concurrency
====================
Concurrency is the number of requests that are processed in parallel. There is
-a global limit and a per-domain limit.
+a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
+can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
+IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
+
+.. note:: The scheduler priority queue :ref:`recommended for broad crawls
+ ` does not support
+ :setting:`CONCURRENT_REQUESTS_PER_IP`.
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
-to increase it will depend on how much CPU you crawler will have available. A
-good starting point is ``100``, but the best way to find out is by doing some
-trials and identifying at what concurrency your Scrapy process gets CPU
-bounded. For optimum performance, you should pick a concurrency where CPU usage
-is at 80-90%.
+to increase it will depend on how much CPU and memory you crawler will have
+available.
-To increase the global concurrency use::
+A good starting point is ``100``::
CONCURRENT_REQUESTS = 100
+But the best way to find out is by doing some trials and identifying at what
+concurrency your Scrapy process gets CPU bounded. For optimum performance, you
+should pick a concurrency where CPU usage is at 80-90%.
+
+Increasing concurrency also increases memory usage. If memory usage is a
+concern, you might need to lower your global concurrency limit accordingly.
+
+
Increase Twisted IO thread pool maximum size
============================================
@@ -85,7 +111,7 @@ When doing broad crawls you are often only interested in the crawl rates you
get and any errors found. These stats are reported by Scrapy when using the
``INFO`` log level. In order to save CPU (and log storage requirements) you
should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be
+production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
@@ -163,3 +189,25 @@ It is turned OFF by default because it has some performance overhead,
and enabling it for focused crawls doesn't make much sense.
.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+.. _broad-crawls-bfo:
+
+Crawl in BFO order
+==================
+
+:ref:`Scrapy crawls in DFO order by default `.
+
+In broad crawls, however, page crawling tends to be faster than page
+processing. As a result, unprocessed early requests stay in memory until the
+final depth is reached, which can significantly increase memory usage.
+
+:ref:`Crawl in BFO order ` instead to save memory.
+
+
+Be mindful of memory leaks
+==========================
+
+If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
+BFO order ` and :ref:`lowering concurrency
+` you should :ref:`debug your memory leaks
+`.
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 97f8311de..5b3cd7e75 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -1,3 +1,5 @@
+.. highlight:: none
+
.. _topics-commands:
=================
@@ -66,7 +68,9 @@ structure by default, similar to this::
The directory where the ``scrapy.cfg`` file resides is known as the *project
root directory*. That file contains the name of the python module that defines
-the project settings. Here is an example::
+the project settings. Here is an example:
+
+.. code-block:: ini
[settings]
default = myproject.settings
@@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
shared by multiple Scrapy projects, each with its own settings module.
In that case, you must define one or more aliases for those settings modules
-under ``[settings]`` in your ``scrapy.cfg`` file::
+under ``[settings]`` in your ``scrapy.cfg`` file:
+
+.. code-block:: ini
[settings]
default = myproject1.settings
@@ -277,6 +283,8 @@ check
Run contract checks.
+.. skip: start
+
Usage examples::
$ scrapy check -l
@@ -294,6 +302,8 @@ Usage examples::
[FAILED] first_spider:parse
>>> Returned 92 requests, expected 0..4
+.. skip: end
+
.. command:: list
list
@@ -461,6 +471,9 @@ Supported options:
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+ This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
+
* ``--pipelines``: process items through pipelines
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
@@ -478,6 +491,8 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
+.. skip: start
+
Usage example::
$ scrapy parse http://www.example.com/ -c parse_item
@@ -492,6 +507,8 @@ Usage example::
# Requests -----------------------------------------------------------------
[]
+.. skip: end
+
.. command:: settings
@@ -570,7 +587,9 @@ Default: ``''`` (empty string)
A module to use for looking up custom Scrapy commands. This is used to add custom
commands for your Scrapy project.
-Example::
+Example:
+
+.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
@@ -585,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.
-The following example adds ``my_command`` command::
+The following example adds ``my_command`` command:
+
+.. skip: next
+
+.. code-block:: python
from setuptools import setup, find_packages
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 70f20d4ed..371ae62d5 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -6,10 +6,6 @@ Spiders Contracts
.. versionadded:: 0.15
-.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
- to minor functionality/API updates. Check the :ref:`release notes ` to
- be notified of updates.
-
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@@ -35,12 +31,20 @@ This callback is tested using three built-in contracts:
.. class:: UrlContract
- This contract (``@url``) sets the sample url used when checking other
+ This contract (``@url``) sets the sample URL used when checking other
contract conditions for this spider. This contract is mandatory. All
callbacks lacking this contract are ignored when running the checks::
@url url
+.. class:: CallbackKeywordArgumentsContract
+
+ This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs `
+ attribute for the sample request. It must be a valid JSON dictionary.
+ ::
+
+ @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+
.. class:: ReturnsContract
This contract (``@returns``) sets lower and upper bounds for the items and
@@ -69,7 +73,7 @@ create and load your own contracts in the project by using the
'myproject.contracts.ItemValidate': 10,
}
-Each contract must inherit from :class:`scrapy.contracts.Contract` and can
+Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
override three methods:
.. module:: scrapy.contracts
@@ -102,9 +106,14 @@ override three methods:
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
+Raise :class:`~scrapy.exceptions.ContractFail` from
+:class:`~scrapy.contracts.Contract.pre_process` or
+:class:`~scrapy.contracts.Contract.post_process` if expectations are not met:
+
+.. autoclass:: scrapy.exceptions.ContractFail
+
Here is a demo contract which checks the presence of a custom header in the
-response received. Raise :class:`scrapy.exceptions.ContractFail` in order to
-get the failures pretty printed::
+response received::
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@@ -120,3 +129,24 @@ get the failures pretty printed::
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
+
+.. _detecting-contract-check-runs:
+
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
+
+ import os
+ import scrapy
+
+ class ExampleSpider(scrapy.Spider):
+ name = 'example'
+
+ def __init__(self):
+ if os.environ.get('SCRAPY_CHECK'):
+ pass # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index f93aa2c72..4b2588518 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -28,16 +28,15 @@ Consider the following scrapy spider below::
item = MyItem()
# populate `item` fields
# and extract item_details_url
- yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+ yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
- def parse_details(self, response):
- item = response.meta['item']
+ def parse_details(self, response, item):
# populate more `item` fields
return item
Basically this is a simple spider which parses two pages of items (the
start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
partially populated item.
@@ -49,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
of the spider at the method level. It has the advantage of being flexible and
simple to use, but does not allow debugging code inside a method.
+.. highlight:: none
+
+.. skip: start
+
In order to see the item scraped from a specific url::
$ scrapy parse --spider=myspider -c parse_item -d 2
@@ -86,6 +89,8 @@ using::
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
+.. skip: end
+
Scrapy Shell
============
@@ -95,13 +100,14 @@ spider, it is of little help to check what happens inside a callback, besides
showing the response received and the output. How to debug the situation when
``parse_details`` sometimes receives no item?
+.. highlight:: python
+
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
from scrapy.shell import inspect_response
- def parse_details(self, response):
- item = response.meta.get('item', None)
+ def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
@@ -134,8 +140,7 @@ Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
- def parse_details(self, response):
- item = response.meta.get('item', None)
+ def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index c1976258d..bf14643be 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -203,7 +203,7 @@ where our quotes are coming from:
First click on the request with the name ``scroll``. On the right
you can now inspect the request. In ``Headers`` you'll find details
about the request headers, such as the URL, the method, the IP-address,
-and so on. We'll ignore the other tabs and click directly on ``Reponse``.
+and so on. We'll ignore the other tabs and click directly on ``Response``.
What you should see in the ``Preview`` pane is the rendered HTML-code,
that is exactly what we saw when we called ``view(response)`` in the
@@ -233,7 +233,7 @@ also request each page to get every quote on the site::
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
page = 1
- start_urls = ['http://quotes.toscrape.com/api/quotes?page=1]
+ start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
def parse(self, response):
data = json.loads(response.text)
@@ -252,9 +252,33 @@ If the handy ``has_next`` element is ``true`` (try loading
`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
page-number greater than 10), we increment the ``page`` attribute
and ``yield`` a new request, inserting the incremented page-number
-into our ``url``.
+into our ``url``.
-You can see that with a few inspections in the `Network`-tool we
+.. _requests-from-curl:
+
+In more complex websites, it could be difficult to easily reproduce the
+requests, as we could need to add ``headers`` or ``cookies`` to make it work.
+In those cases you can export the requests in `cURL `_
+format, by right-clicking on each of them in the network tool and using the
+:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
+request::
+
+ from scrapy import Request
+
+ request = Request.from_curl(
+ "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
+ "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
+ "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
+ "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
+ "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
+ "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
+ "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
+
+Alternatively, if you want to know the arguments needed to recreate that
+request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments.
+
+As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite
daunting and pages can be very complex, but it (mostly) boils down
@@ -262,7 +286,7 @@ to identifying the correct request and replicating it in your spider.
.. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
.. _quotes.toscrape.com: http://quotes.toscrape.com
-.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8dbe249fa..ae6d41809 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied.
If you want to disable a built-in middleware (the ones defined in
:setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
-in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
+in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
as its value. For example, if you want to disable the user-agent middleware::
DOWNLOADER_MIDDLEWARES = {
@@ -52,11 +52,17 @@ as its value. For example, if you want to disable the user-agent middleware::
Finally, keep in mind that some middlewares may need to be enabled through a
particular setting. See each middleware documentation for more info.
+.. _topics-downloader-middleware-custom:
+
Writing your own downloader middleware
======================================
-Each middleware component is a Python class that defines one or
-more of the following methods:
+Each downloader middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings `.
.. module:: scrapy.downloadermiddlewares
@@ -342,10 +348,9 @@ HttpCacheMiddleware
* :ref:`httpcache-storage-fs`
* :ref:`httpcache-storage-dbm`
- * :ref:`httpcache-storage-leveldb`
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
- setting. Or you can also implement your own storage backend.
+ setting. Or you can also :ref:`implement your own storage backend. `
Scrapy ships with two HTTP cache policies:
@@ -357,26 +362,27 @@ HttpCacheMiddleware
.. reqmeta:: dont_cache
- You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
+ You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``.
+
+.. module:: scrapy.extensions.httpcache
+ :noindex:
.. _httpcache-policy-dummy:
Dummy policy (default)
~~~~~~~~~~~~~~~~~~~~~~
-This policy has no awareness of any HTTP Cache-Control directives.
-Every request and its corresponding response are cached. When the same
-request is seen again, the response is returned without transferring
-anything from the Internet.
+.. class:: DummyPolicy
-The Dummy policy is useful for testing spiders faster (without having
-to wait for downloads every time) and for trying your spider offline,
-when an Internet connection is not available. The goal is to be able to
-"replay" a spider run *exactly as it ran before*.
+ This policy has no awareness of any HTTP Cache-Control directives.
+ Every request and its corresponding response are cached. When the same
+ request is seen again, the response is returned without transferring
+ anything from the Internet.
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy``
+ The Dummy policy is useful for testing spiders faster (without having
+ to wait for downloads every time) and for trying your spider offline,
+ when an Internet connection is not available. The goal is to be able to
+ "replay" a spider run *exactly as it ran before*.
.. _httpcache-policy-rfc2616:
@@ -384,45 +390,44 @@ In order to use this policy, set:
RFC2616 policy
~~~~~~~~~~~~~~
-This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
-Cache-Control awareness, aimed at production and used in continuous
-runs to avoid downloading unmodified data (to save bandwidth and speed up crawls).
+.. class:: RFC2616Policy
-what is implemented:
+ This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
+ Cache-Control awareness, aimed at production and used in continuous
+ runs to avoid downloading unmodified data (to save bandwidth and speed up
+ crawls).
-* Do not attempt to store responses/requests with `no-store` cache-control directive set
-* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses
-* Compute freshness lifetime from `max-age` cache-control directive
-* Compute freshness lifetime from `Expires` response header
-* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox)
-* Compute current age from `Age` response header
-* Compute current age from `Date` header
-* Revalidate stale responses based on `Last-Modified` response header
-* Revalidate stale responses based on `ETag` response header
-* Set `Date` header for any received response missing it
-* Support `max-stale` cache-control directive in requests
+ What is implemented:
- This allows spiders to be configured with the full RFC2616 cache policy,
- but avoid revalidation on a request-by-request basis, while remaining
- conformant with the HTTP spec.
+ * Do not attempt to store responses/requests with ``no-store`` cache-control directive set
+ * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
+ * Compute freshness lifetime from ``max-age`` cache-control directive
+ * Compute freshness lifetime from ``Expires`` response header
+ * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
+ * Compute current age from ``Age`` response header
+ * Compute current age from ``Date`` header
+ * Revalidate stale responses based on ``Last-Modified`` response header
+ * Revalidate stale responses based on ``ETag`` response header
+ * Set ``Date`` header for any received response missing it
+ * Support ``max-stale`` cache-control directive in requests
- Example:
+ This allows spiders to be configured with the full RFC2616 cache policy,
+ but avoid revalidation on a request-by-request basis, while remaining
+ conformant with the HTTP spec.
- Add `Cache-Control: max-stale=600` to Request headers to accept responses that
- have exceeded their expiration time by no more than 600 seconds.
+ Example:
- See also: RFC2616, 14.9.3
+ Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
+ have exceeded their expiration time by no more than 600 seconds.
-what is missing:
+ See also: RFC2616, 14.9.3
-* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
-* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
-* ... probably others ..
+ What is missing:
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy``
+ * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+ * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+ * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
+ * ... probably others ..
.. _httpcache-storage-fs:
@@ -430,67 +435,102 @@ In order to use this policy, set:
Filesystem storage backend (default)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-File system storage backend is available for the HTTP cache middleware.
+.. class:: FilesystemCacheStorage
-In order to use this storage backend, set:
+ File system storage backend is available for the HTTP cache middleware.
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage``
+ Each request/response pair is stored in a different directory containing
+ the following files:
-Each request/response pair is stored in a different directory containing
-the following files:
+ * ``request_body`` - the plain request body
- * ``request_body`` - the plain request body
- * ``request_headers`` - the request headers (in raw HTTP format)
- * ``response_body`` - the plain response body
- * ``response_headers`` - the request headers (in raw HTTP format)
- * ``meta`` - some metadata of this cache resource in Python ``repr()`` format
- (grep-friendly format)
- * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
- efficient deserialization
+ * ``request_headers`` - the request headers (in raw HTTP format)
-The directory name is made from the request fingerprint (see
-``scrapy.utils.request.fingerprint``), and one level of subdirectories is
-used to avoid creating too many files into the same directory (which is
-inefficient in many file systems). An example directory could be::
+ * ``response_body`` - the plain response body
- /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+ * ``response_headers`` - the request headers (in raw HTTP format)
+
+ * ``meta`` - some metadata of this cache resource in Python ``repr()``
+ format (grep-friendly format)
+
+ * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
+ efficient deserialization
+
+ The directory name is made from the request fingerprint (see
+ ``scrapy.utils.request.fingerprint``), and one level of subdirectories is
+ used to avoid creating too many files into the same directory (which is
+ inefficient in many file systems). An example directory could be::
+
+ /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
.. _httpcache-storage-dbm:
DBM storage backend
~~~~~~~~~~~~~~~~~~~
-.. versionadded:: 0.13
+.. class:: DbmCacheStorage
-A DBM_ storage backend is also available for the HTTP cache middleware.
+ .. versionadded:: 0.13
-By default, it uses the anydbm_ module, but you can change it with the
-:setting:`HTTPCACHE_DBM_MODULE` setting.
+ A DBM_ storage backend is also available for the HTTP cache middleware.
-In order to use this storage backend, set:
+ By default, it uses the :mod:`dbm`, but you can change it with the
+ :setting:`HTTPCACHE_DBM_MODULE` setting.
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage``
+.. _httpcache-storage-custom:
-.. _httpcache-storage-leveldb:
+Writing your own storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-LevelDB storage backend
-~~~~~~~~~~~~~~~~~~~~~~~
+You can implement a cache storage backend by creating a Python class that
+defines the methods described below.
-.. versionadded:: 0.23
+.. module:: scrapy.extensions.httpcache
-A LevelDB_ storage backend is also available for the HTTP cache middleware.
+.. class:: CacheStorage
-This backend is not recommended for development because only one process can
-access LevelDB databases at the same time, so you can't run a crawl and open
-the scrapy shell in parallel for the same spider.
+ .. method:: open_spider(spider)
-In order to use this storage backend:
+ This method gets called after a spider has been opened for crawling. It handles
+ the :signal:`open_spider ` signal.
-* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
-* install `LevelDB python bindings`_ like ``pip install leveldb``
+ :param spider: the spider which has been opened
+ :type spider: :class:`~scrapy.spiders.Spider` object
-.. _LevelDB: https://github.com/google/leveldb
-.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
+ .. method:: close_spider(spider)
+
+ This method gets called after a spider has been closed. It handles
+ the :signal:`close_spider ` signal.
+
+ :param spider: the spider which has been closed
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ .. method:: retrieve_response(spider, request)
+
+ Return response if present in cache, or ``None`` otherwise.
+
+ :param spider: the spider which generated the request
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ :param request: the request to find cached response for
+ :type request: :class:`~scrapy.http.Request` object
+
+ .. method:: store_response(spider, request, response)
+
+ Store the given response in the cache.
+
+ :param spider: the spider for which the response is intended
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ :param request: the corresponding request the spider generated
+ :type request: :class:`~scrapy.http.Request` object
+
+ :param response: the response to store in the cache
+ :type response: :class:`~scrapy.http.Response` object
+
+In order to use your storage backend, set:
+
+* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class.
HTTPCache middleware settings
@@ -586,7 +626,7 @@ HTTPCACHE_DBM_MODULE
.. versionadded:: 0.13
-Default: ``'anydbm'``
+Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
`. This setting is specific to the DBM backend.
@@ -626,12 +666,12 @@ Default: ``False``
If enabled, will cache pages unconditionally.
A spider may wish to have all responses available in the cache, for
-future use with `Cache-Control: max-stale`, for instance. The
+future use with ``Cache-Control: max-stale``, for instance. The
DummyPolicy caches all responses but never revalidates them, and
sometimes a more nuanced policy is desirable.
-This setting still respects `Cache-Control: no-store` directives in responses.
-If you don't want that, filter `no-store` out of the Cache-Control headers in
+This setting still respects ``Cache-Control: no-store`` directives in responses.
+If you don't want that, filter ``no-store`` out of the Cache-Control headers in
responses you feedto the cache middleware.
.. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
@@ -729,6 +769,17 @@ RedirectMiddleware
The urls which the request goes through (while being redirected) can be found
in the ``redirect_urls`` :attr:`Request.meta ` key.
+.. reqmeta:: redirect_reasons
+
+The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
+``redirect_reasons`` :attr:`Request.meta ` key. For
+example: ``[301, 302, 307, 'meta refresh']``.
+
+The format of a reason depends on the middleware that handled the corresponding
+redirect. For example, :class:`RedirectMiddleware` indicates the triggering
+response status code as an integer, while :class:`MetaRefreshMiddleware`
+always uses the ``'meta refresh'`` string as reason.
+
The :class:`RedirectMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@@ -790,10 +841,12 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
settings (see the settings documentation for more info):
* :setting:`METAREFRESH_ENABLED`
+* :setting:`METAREFRESH_IGNORE_TAGS`
* :setting:`METAREFRESH_MAXDELAY`
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
MetaRefreshMiddleware settings
@@ -810,6 +863,15 @@ Default: ``True``
Whether the Meta Refresh middleware will be enabled.
+.. setting:: METAREFRESH_IGNORE_TAGS
+
+METAREFRESH_IGNORE_TAGS
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``['script', 'noscript']``
+
+Meta tags within these tags are ignored.
+
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY
@@ -834,8 +896,6 @@ RetryMiddleware
Failed pages are collected on the scraping process and rescheduled at the
end, once the spider has finished crawling all regular (non failed) pages.
-Once there are no more failed pages to retry, this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
The :class:`RetryMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@@ -882,7 +942,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
RETRY_HTTP_CODES
^^^^^^^^^^^^^^^^
-Default: ``[500, 502, 503, 504, 522, 524, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408, 429]``
Which HTTP response codes to retry. Other errors (DNS lookup issues,
connections lost, etc) are always retried.
@@ -908,6 +968,24 @@ RobotsTxtMiddleware
To make sure Scrapy respects robots.txt make sure the middleware is enabled
and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+ The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+ user agent string to use for matching in the robots.txt_ file. If it
+ is ``None``, the User-Agent header you are sending with the request or the
+ :setting:`USER_AGENT` setting (in that order) will be used for determining
+ the user agent to use in the robots.txt_ file.
+
+ This middleware has to be combined with a robots.txt_ parser.
+
+ Scrapy ships with support for the following robots.txt_ parsers:
+
+ * :ref:`Protego ` (default)
+ * :ref:`RobotFileParser `
+ * :ref:`Reppy `
+ * :ref:`Robotexclusionrulesparser `
+
+ You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+ setting. Or you can also :ref:`implement support for a new parser `.
+
.. reqmeta:: dont_obey_robotstxt
If :attr:`Request.meta ` has
@@ -915,6 +993,129 @@ If :attr:`Request.meta ` has
the request will be ignored by this middleware even if
:setting:`ROBOTSTXT_OBEY` is enabled.
+Parsers vary in several aspects:
+
+* Language of implementation
+
+* Supported specification
+
+* Support for wildcard matching
+
+* Usage of `length based rule `_:
+ in particular for ``Allow`` and ``Disallow`` directives, where the most
+ specific rule based on the length of the path trumps the less specific
+ (shorter) rule
+
+Performance comparison of different parsers is available at `the following link
+`_.
+
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+Based on `Protego `_:
+
+* implemented in Python
+
+* is compliant with `Google's Robots.txt Specification
+ `_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Scrapy uses this parser by default.
+
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+Based on `RobotFileParser
+`_:
+
+* is Python's built-in robots.txt_ parser
+
+* is compliant with `Martijn Koster's 1996 draft specification
+ `_
+
+* lacks support for wildcard matching
+
+* doesn't use the length based rule
+
+It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+Based on `Reppy `_:
+
+* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
+ `_
+
+* is compliant with `Martijn Koster's 1996 draft specification
+ `_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Native implementation, provides better speed than Protego.
+
+In order to use this parser:
+
+* Install `Reppy `_ by running ``pip install reppy``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+ ``scrapy.robotstxt.ReppyRobotParser``
+
+.. _rerp-parser:
+
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Based on `Robotexclusionrulesparser `_:
+
+* implemented in Python
+
+* is compliant with `Martijn Koster's 1996 draft specification
+ `_
+
+* supports wildcard matching
+
+* doesn't use the length based rule
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser `_ by running
+ ``pip install robotexclusionrulesparser``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+ ``scrapy.robotstxt.RerpRobotParser``
+
+.. _support-for-new-robots-parser:
+
+Implementing support for a new parser
+-------------------------------------
+
+You can implement support for a new robots.txt_ parser by subclassing
+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
+implementing the methods described below.
+
+.. module:: scrapy.robotstxt
+ :synopsis: robots.txt parser interface and implementations
+
+.. autoclass:: RobotParser
+ :members:
+
+.. _robots.txt: http://www.robotstxt.org/
DownloaderStats
---------------
@@ -940,7 +1141,7 @@ UserAgentMiddleware
Middleware that allows spiders to override the default user agent.
- In order for a spider to override the default user agent, its `user_agent`
+ In order for a spider to override the default user agent, its ``user_agent``
attribute must be set.
.. _ajaxcrawl-middleware:
@@ -1001,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
.. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
new file mode 100644
index 000000000..8334ddcec
--- /dev/null
+++ b/docs/topics/dynamic-content.rst
@@ -0,0 +1,253 @@
+.. _topics-dynamic-content:
+
+====================================
+Selecting dynamically-loaded content
+====================================
+
+Some webpages show the desired data when you load them in a web browser.
+However, when you download them using Scrapy, you cannot reach the desired data
+using :ref:`selectors `.
+
+When this happens, the recommended approach is to
+:ref:`find the data source ` and extract the data
+from it.
+
+If you fail to do that, and you can nonetheless access the desired data through
+the :ref:`DOM ` from your web browser, see
+:ref:`topics-javascript-rendering`.
+
+.. _topics-finding-data-source:
+
+Finding the data source
+=======================
+
+To extract the desired data, you must first find its source location.
+
+If the data is in a non-text-based format, such as an image or a PDF document,
+use the :ref:`network tool ` of your web browser to find
+the corresponding request, and :ref:`reproduce it
+`.
+
+If your web browser lets you select the desired data as text, the data may be
+defined in embedded JavaScript code, or loaded from an external resource in a
+text-based format.
+
+In that case, you can use a tool like wgrep_ to find the URL of that resource.
+
+If the data turns out to come from the original URL itself, you must
+:ref:`inspect the source code of the webpage ` to
+determine where the data is located.
+
+If the data comes from a different URL, you will need to :ref:`reproduce the
+corresponding request `.
+
+.. _topics-inspecting-source:
+
+Inspecting the source code of a webpage
+=======================================
+
+Sometimes you need to inspect the source code of a webpage (not the
+:ref:`DOM `) to determine where some desired data is located.
+
+Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
+by Scrapy::
+
+ scrapy fetch --nolog https://example.com > response.html
+
+If the desired data is in embedded JavaScript code within a ````
+element, see :ref:`topics-parsing-javascript`.
+
+If you cannot find the desired data, first make sure it’s not just Scrapy:
+download the webpage with an HTTP client like curl_ or wget_ and see if the
+information can be found in the response they get.
+
+If they get a response with the desired data, modify your Scrapy
+:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+example, try using the same user-agent string (:setting:`USER_AGENT`) or the
+same :attr:`~scrapy.http.Request.headers`.
+
+If they also get a response without the desired data, you’ll need to take
+steps to make your request more similar to that of the web browser. See
+:ref:`topics-reproducing-requests`.
+
+.. _topics-reproducing-requests:
+
+Reproducing requests
+====================
+
+Sometimes we need to reproduce a request the way our web browser performs it.
+
+Use the :ref:`network tool ` of your web browser to see
+how your web browser performs the desired request, and try to reproduce that
+request with Scrapy.
+
+It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+method and URL. However, you may also need to reproduce the body, headers and
+form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+
+As all major browsers allow to export the requests in `cURL
+`_ format, Scrapy incorporates the method
+:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
+:class:`~scrapy.http.Request` from a cURL command. To get more information
+visit :ref:`request from curl ` inside the network
+tool section.
+
+Once you get the expected response, you can :ref:`extract the desired data from
+it `.
+
+You can reproduce any request with Scrapy. However, some times reproducing all
+necessary requests may not seem efficient in developer time. If that is your
+case, and crawling speed is not a major concern for you, you can alternatively
+consider :ref:`JavaScript pre-rendering `.
+
+If you get the expected response `sometimes`, but not always, the issue is
+probably not your request, but the target server. The target server might be
+buggy, overloaded, or :ref:`banning ` some of your requests.
+
+.. _topics-handling-response-formats:
+
+Handling different response formats
+===================================
+
+Once you have a response with the desired data, how you extract the desired
+data from it depends on the type of response:
+
+- If the response is HTML or XML, use :ref:`selectors
+ ` as usual.
+
+- If the response is JSON, use `json.loads`_ to load the desired data from
+ :attr:`response.text `::
+
+ data = json.loads(response.text)
+
+ If the desired data is inside HTML or XML code embedded within JSON data,
+ you can load that HTML or XML code into a
+ :class:`~scrapy.selector.Selector` and then
+ :ref:`use it ` as usual::
+
+ selector = Selector(data['html'])
+
+- If the response is JavaScript, or HTML with a ```` element
+ containing the desired data, see :ref:`topics-parsing-javascript`.
+
+- If the response is CSS, use a `regular expression`_ to extract the desired
+ data from :attr:`response.text `.
+
+.. _topics-parsing-images:
+
+- If the response is an image or another format based on images (e.g. PDF),
+ read the response as bytes from
+ :attr:`response.body ` and use an OCR
+ solution to extract the desired data as text.
+
+ For example, you can use pytesseract_. To read a table from a PDF,
+ `tabula-py`_ may be a better choice.
+
+- If the response is SVG, or HTML with embedded SVG containing the desired
+ data, you may be able to extract the desired data using
+ :ref:`selectors `, since SVG is based on XML.
+
+ Otherwise, you might need to convert the SVG code into a raster image, and
+ :ref:`handle that raster image `.
+
+.. _topics-parsing-javascript:
+
+Parsing JavaScript code
+=======================
+
+If the desired data is hardcoded in JavaScript, you first need to get the
+JavaScript code:
+
+- If the JavaScript code is in a JavaScript file, simply read
+ :attr:`response.text `.
+
+- If the JavaScript code is within a ```` element of an HTML page,
+ use :ref:`selectors ` to extract the text within that
+ ```` element.
+
+Once you have a string with the JavaScript code, you can extract the desired
+data from it:
+
+- You might be able to use a `regular expression`_ to extract the desired
+ data in JSON format, which you can then parse with `json.loads`_.
+
+ For example, if the JavaScript code contains a separate line like
+ ``var data = {"field": "value"};`` you can extract that data as follows::
+
+ >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+ >>> json_data = response.css('script::text').re_first(pattern)
+ >>> json.loads(json_data)
+ {'field': 'value'}
+
+- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
+ that you can parse using :ref:`selectors `.
+
+ For example, if the JavaScript code contains
+ ``var data = {field: "value"};`` you can extract that data as follows::
+
+ >>> import js2xml
+ >>> import lxml.etree
+ >>> from parsel import Selector
+ >>> javascript = response.css('script::text').get()
+ >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+ >>> selector = Selector(text=xml)
+ >>> selector.css('var[name="data"]').get()
+ ''
+
+.. _topics-javascript-rendering:
+
+Pre-rendering JavaScript
+========================
+
+On webpages that fetch data from additional requests, reproducing those
+requests that contain the desired data is the preferred approach. The effort is
+often worth the result: structured, complete data with minimum parsing time and
+network transfer.
+
+However, sometimes it can be really hard to reproduce certain requests. Or you
+may need something that no request can give you, such as a screenshot of a
+webpage as seen in a web browser.
+
+In these cases use the Splash_ JavaScript-rendering service, along with
+`scrapy-splash`_ for seamless integration.
+
+Splash returns as HTML the :ref:`DOM ` of a webpage, so that
+you can parse it with :ref:`selectors `. It provides great
+flexibility through configuration_ or scripting_.
+
+If you need something beyond what Splash offers, such as interacting with the
+DOM on-the-fly from Python code instead of using a previously-written script,
+or handling multiple web browser windows, you might need to
+:ref:`use a headless browser ` instead.
+
+.. _configuration: https://splash.readthedocs.io/en/stable/api.html
+.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
+
+.. _topics-headless-browsing:
+
+Using a headless browser
+========================
+
+A `headless browser`_ is a special web browser that provides an API for
+automation.
+
+The easiest way to use a headless browser with Scrapy is to use Selenium_,
+along with `scrapy-selenium`_ for seamless integration.
+
+
+.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _curl: https://curl.haxx.se/
+.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _js2xml: https://github.com/scrapinghub/js2xml
+.. _json.loads: https://docs.python.org/library/json.html#json.loads
+.. _pytesseract: https://github.com/madmaze/pytesseract
+.. _regular expression: https://docs.python.org/library/re.html
+.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+.. _Selenium: https://www.seleniumhq.org/
+.. _Splash: https://github.com/scrapinghub/splash
+.. _tabula-py: https://github.com/chezou/tabula-py
+.. _wget: https://www.gnu.org/software/wget/
+.. _wgrep: https://github.com/stav/wgrep
\ No newline at end of file
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 949cdc638..72bf52227 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -9,19 +9,19 @@ Sending e-mail
Although Python makes sending e-mails relatively easy via the `smtplib`_
library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
-interfering with the non-blocking IO of the crawler. It also provides a
-simple API for sending attachments and it's very easy to configure, with a few
-:ref:`settings `.
+easy to use and it's implemented using :doc:`Twisted non-blocking IO
+`, to avoid interfering with the non-blocking
+IO of the crawler. It also provides a simple API for sending attachments and
+it's very easy to configure, with a few :ref:`settings
+`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
-the standard constructor::
+the standard ``__init__`` method::
from scrapy.mail import MailSender
mailer = MailSender()
@@ -39,7 +39,8 @@ MailSender class reference
==========================
MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework.
+uses :doc:`Twisted non-blocking IO `, like the
+rest of the framework.
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
@@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
Mail settings
=============
-These settings define the default constructor values of the :class:`MailSender`
+These settings define the default ``__init__`` method values of the :class:`MailSender`
class, and can be used to configure e-mail notifications in your project without
writing any code (for those extensions and code that uses :class:`MailSender`).
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 95f7920f8..b8d898022 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -51,7 +51,6 @@ value of one of their fields::
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
exporter.finish_exporting()
- exporter.file.close()
def _exporter_for_item(self, item):
year = item['year']
@@ -88,8 +87,8 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
-If you use :class:`~.Item` you can declare a serializer in the
-:ref:`field metadata `. The serializer must be
+If you use :class:`~.Item` you can declare a serializer in the
+:ref:`field metadata `. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
@@ -145,7 +144,7 @@ BaseItemExporter
defining what fields to export, whether to export empty fields, or which
encoding to use.
- These features can be configured through the constructor arguments which
+ These features can be configured through the ``__init__`` method arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
@@ -165,9 +164,9 @@ BaseItemExporter
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
- :param field: the field being serialized. If a raw dict is being
+ :param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
- :type field: :class:`~scrapy.item.Field` object or an empty dict
+ :type field: :class:`~scrapy.item.Field` object or an empty dict
:param name: the name of the field being serialized
:type name: str
@@ -223,6 +222,12 @@ BaseItemExporter
* ``indent<=0`` each item on its own line, no indentation
* ``indent>0`` each item on its own line, indented with the provided numeric value
+PythonItemExporter
+------------------
+
+.. autoclass:: PythonItemExporter
+
+
.. highlight:: none
XmlItemExporter
@@ -241,8 +246,8 @@ XmlItemExporter
:param item_element: The name of each item element in the exported XML.
:type item_element: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -301,9 +306,9 @@ CsvItemExporter
multi-valued fields, if found.
:type include_headers_line: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor, and the leftover arguments to the
- `csv.writer`_ constructor, so you can use any `csv.writer` constructor
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
+ `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
argument to customize this exporter.
A typical output of this exporter would be::
@@ -329,8 +334,8 @@ PickleItemExporter
For more information, refer to the `pickle module documentation`_.
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
@@ -346,8 +351,8 @@ PprintItemExporter
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -362,10 +367,10 @@ JsonItemExporter
.. class:: JsonItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing all
- objects as a list of objects. The additional constructor arguments are
- passed to the :class:`BaseItemExporter` constructor, and the leftover
- arguments to the `JSONEncoder`_ constructor, so you can use any
- `JSONEncoder`_ constructor argument to customize this exporter.
+ objects as a list of objects. The additional ``__init__`` method arguments are
+ passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
+ arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
+ `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -393,10 +398,10 @@ JsonLinesItemExporter
.. class:: JsonLinesItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing one
- JSON-encoded item per line. The additional constructor arguments are passed
- to the :class:`BaseItemExporter` constructor, and the leftover arguments to
- the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
- constructor argument to customize this exporter.
+ JSON-encoded item per line. The additional ``__init__`` method arguments are passed
+ to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
+ the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
+ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -410,3 +415,8 @@ JsonLinesItemExporter
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+
+MarshalItemExporter
+-------------------
+
+.. autoclass:: MarshalItemExporter
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index c421a5e05..0a7455ec9 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -19,7 +19,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
-`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
+``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
@@ -28,7 +28,7 @@ Loading & activating extensions
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class constructor (``__init__`` method).
+code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@@ -183,7 +183,7 @@ Telnet console extension
.. module:: scrapy.extensions.telnet
:synopsis: Telnet console
-.. class:: scrapy.extensions.telnet.TelnetConsole
+.. class:: TelnetConsole
Provides a telnet console for getting into a Python interpreter inside the
currently running Scrapy process, which can be very useful for debugging.
@@ -200,7 +200,7 @@ Memory usage extension
.. module:: scrapy.extensions.memusage
:synopsis: Memory usage extension
-.. class:: scrapy.extensions.memusage.MemoryUsage
+.. class:: MemoryUsage
.. note:: This extension does not work in Windows.
@@ -228,7 +228,7 @@ Memory debugger extension
.. module:: scrapy.extensions.memdebug
:synopsis: Memory debugger extension
-.. class:: scrapy.extensions.memdebug.MemoryDebugger
+.. class:: MemoryDebugger
An extension for debugging memory usage. It collects information about:
@@ -244,7 +244,7 @@ Close spider extension
.. module:: scrapy.extensions.closespider
:synopsis: Close spider extension
-.. class:: scrapy.extensions.closespider.CloseSpider
+.. class:: CloseSpider
Closes a spider automatically when some conditions are met, using a specific
closing reason for each condition.
@@ -317,7 +317,7 @@ StatsMailer extension
.. module:: scrapy.extensions.statsmailer
:synopsis: StatsMailer extension
-.. class:: scrapy.extensions.statsmailer.StatsMailer
+.. class:: StatsMailer
This simple extension can be used to send a notification e-mail every time a
domain has finished scraping, including the Scrapy stats collected. The email
@@ -333,7 +333,7 @@ Debugging extensions
Stack trace dump extension
~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. class:: scrapy.extensions.debug.StackTraceDump
+.. class:: StackTraceDump
Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
signal is received. The information dumped is the following:
@@ -362,13 +362,13 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
Debugger extension
~~~~~~~~~~~~~~~~~~
-.. class:: scrapy.extensions.debug.Debugger
+.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
-For more info see `Debugging in Python`.
+For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (ie. not Windows).
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index b64dbfbfd..7481b1a99 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -99,12 +99,12 @@ The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
-or boto_ library is installed (Scrapy supports boto_ only on Python 2).
+library is installed.
.. _topics-feed-uri-params:
@@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
+FTP supports two different connection modes: `active or passive
+`_. Scrapy uses the passive connection
+mode by default. To use the active connection mode instead, set the
+:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
.. _topics-feed-storage-s3:
S3
@@ -177,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
+ * Required external libraries: `botocore`_
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@@ -185,6 +190,10 @@ passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
+You can also define a custom ACL for exported feeds using this setting:
+
+ * :setting:`FEED_STORAGE_S3_ACL`
+
.. _topics-feed-storage-stdout:
Standard output
@@ -205,6 +214,8 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_URI` (mandatory)
* :setting:`FEED_FORMAT`
* :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_FTP_ACTIVE`
+ * :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
@@ -302,6 +313,30 @@ Default: ``{}``
A dict containing additional feed storage backends supported by your project.
The keys are URI schemes and the values are paths to storage classes.
+.. setting:: FEED_STORAGE_FTP_ACTIVE
+
+FEED_STORAGE_FTP_ACTIVE
+-----------------------
+
+Default: ``False``
+
+Whether to use the active connection mode when exporting feeds to an FTP server
+(``True``) or use the passive connection mode instead (``False``, default).
+
+For information about FTP connection modes, see `What is the difference between
+active and passive FTP? `_.
+
+.. setting:: FEED_STORAGE_S3_ACL
+
+FEED_STORAGE_S3_ACL
+-------------------
+
+Default: ``''`` (empty string)
+
+A string containing a custom ACL for feeds exported to Amazon S3 by your project.
+
+For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs.
+
.. setting:: FEED_STORAGES_BASE
FEED_STORAGES_BASE
@@ -364,5 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto: https://github.com/boto/boto
.. _botocore: https://github.com/boto/botocore
+.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index fae18200a..cdc4953c2 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
- (or any descendant class) object, return a `Twisted Deferred`_ or raise
+ (or any descendant class) object, return a
+ :class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
@@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
:type crawler: :class:`~scrapy.crawler.Crawler` object
-.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
-
Item pipeline example
=====================
@@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
Take screenshot of item
-----------------------
-This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+This example demonstrates how to return a
+:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
@@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
-.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
Duplicates filter
-----------------
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index ae44aecd3..cdf60208e 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -16,12 +16,12 @@ especially in a larger project with many spiders.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields.
+their available fields.
-Various Scrapy components use extra information provided by Items:
+Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks
+tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@@ -40,6 +40,7 @@ objects. Here is an example::
name = scrapy.Field()
price = scrapy.Field()
stock = scrapy.Field()
+ tags = scrapy.Field()
last_updated = scrapy.Field(serializer=str)
.. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@@ -155,19 +156,44 @@ To access all populated values, just use the typical `dict API`_::
>>> product.items()
[('price', 1000), ('name', 'Desktop PC')]
+
+.. _copying-items:
+
+Copying items
+-------------
+
+To copy an item, you must first decide whether you want a shallow copy or a
+deep copy.
+
+If your item contains mutable_ values like lists or dictionaries, a shallow
+copy will keep references to the same mutable values across all different
+copies.
+
+.. _mutable: https://docs.python.org/glossary.html#term-mutable
+
+For example, if you have an item with a list of tags, and you create a shallow
+copy of that item, both the original item and the copy have the same list of
+tags. Adding a tag to the list of one of the items will add the tag to the
+other item as well.
+
+If that is not the desired behavior, use a deep copy instead.
+
+See the `documentation of the copy module`_ for more information.
+
+.. _documentation of the copy module: https://docs.python.org/library/copy.html
+
+To create a shallow copy of an item, you can either call
+:meth:`~scrapy.item.Item.copy` on an existing item
+(``product2 = product.copy()``) or instantiate your item class from an existing
+item (``product2 = Product(product)``).
+
+To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+(``product2 = product.deepcopy()``).
+
+
Other common tasks
------------------
-Copying items::
-
- >>> product2 = Product(product)
- >>> print(product2)
- Product(name='Desktop PC', price=1000)
-
- >>> product3 = product2.copy()
- >>> print(product3)
- Product(name='Desktop PC', price=1000)
-
Creating dicts from items::
>>> dict(product) # create a dict from all populated values
@@ -211,8 +237,12 @@ Item objects
Return a new Item optionally initialized from the given argument.
- Items replicate the standard `dict API`_, including its constructor. The
- only additional attribute provided by Items is:
+ Items replicate the standard `dict API`_, including its ``__init__`` method, and
+ also provide the following additional API members:
+
+ .. automethod:: copy
+
+ .. automethod:: deepcopy
.. attribute:: fields
@@ -237,3 +267,9 @@ Field objects
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index ea684b4cf..f5542495b 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -71,33 +71,11 @@ on cookies.
Request serialization
---------------------
-Requests must be serializable by the `pickle` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
- def some_callback(self, response):
- somearg = 'test'
- return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
-
- def other_callback(self, response, somearg):
- print("the argument passed is: %s" % somearg)
-
-But this will::
-
- def some_callback(self, response):
- somearg = 'test'
- return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
-
- def other_callback(self, response):
- somearg = response.meta['somearg']
- print("the argument passed is: %s" % somearg)
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+runnning :class:`~scrapy.spiders.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index af14d14e8..793636f59 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -27,10 +27,11 @@ Common causes of memory leaks
It happens quite often (sometimes by accident, sometimes on purpose) that the
Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
In big projects, the spiders are typically written by different people and some
of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
By default Scrapy keeps the request queue in memory; it includes
:class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
While not necessarily a leak, this can take a lot of memory. Enabling
:ref:`persistent job queue ` could help keeping memory usage
in control.
@@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
- callback=self.parse, meta={referer: response})
+ callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
ties the response lifetime to the requests' one, and that would definitely
@@ -258,7 +260,7 @@ knowledge about Python internals. For more info about Guppy, refer to the
Debugging memory leaks with muppy
=================================
-If you're using Python 3, you can use muppy from `Pympler`_.
+You can use muppy from `Pympler`_.
.. _Pympler: https://pypi.org/project/Pympler/
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index f40a36d31..713a94e10 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -93,6 +93,12 @@ LxmlLinkExtractor
Has the same behaviour as ``restrict_xpaths``.
:type restrict_css: str or list
+ :param restrict_text: a single regular expression (or list of regular expressions)
+ that the link's text must match in order to be extracted. If not
+ given (or empty), it will match all links. If a list of regular expressions is
+ given, the link will be extracted if it matches at least one.
+ :type restrict_text: a regular expression (or list of)
+
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
:type tags: str or list
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index f3b6aa4a1..de3f38023 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader constructor
+which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
@@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
the same item field; the Item Loader will know how to "join" those values later
using a proper processing function.
+.. note:: Collected data is internally stored as lists,
+ allowing to add several values to the same field.
+ If an ``item`` argument is passed when creating a loader,
+ each of the item's values will be stored as-is if it's already
+ an iterable, or wrapped with a list if it's a single value.
+
Here is a typical Item Loader usage in a :ref:`Spider `, using
the :ref:`Product item ` declared in the :ref:`Items
chapter `::
@@ -128,28 +134,14 @@ So what happens is:
It's worth noticing that processors are just callable objects, which are called
with the data to be parsed, and return a parsed value. So you can use any
function as input or output processor. The only requirement is that they must
-accept one (and only one) positional argument, which will be an iterator.
+accept one (and only one) positional argument, which will be an iterable.
-.. note:: Both input and output processors must receive an iterator as their
+.. note:: Both input and output processors must receive an iterable as their
first argument. The output of those functions can be anything. The result of
input processors will be appended to an internal list (in the Loader)
containing the collected values (for that field). The result of the output
processors is the value that will be finally assigned to the item.
-If you want to use a plain function as a processor, make sure it receives
-``self`` as the first argument::
-
- def lowercase_processor(self, values):
- for v in values:
- yield v.lower()
-
- class MyItemLoader(ItemLoader):
- name_in = lowercase_processor
-
-This is because whenever a function is assigned as a class variable, it becomes
-a method and would be passed the instance as the the first argument when being
-called. See `this answer on stackoverflow`_ for more details.
-
The other thing you need to keep in mind is that the values returned by input
processors are collected internally (in lists) and then passed to output
processors to populate the fields.
@@ -157,7 +149,7 @@ processors to populate the fields.
Last, but not least, Scrapy comes with some :ref:`commonly used processors
` built-in for convenience.
-.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
+
Declaring Item Loaders
======================
@@ -265,7 +257,7 @@ There are several ways to modify Item Loader context values:
loader.context['unit'] = 'cm'
2. On Item Loader instantiation (the keyword arguments of Item Loader
- constructor are stored in the Item Loader context)::
+ ``__init__`` method are stored in the Item Loader context)::
loader = ItemLoader(product, unit='cm')
@@ -286,7 +278,7 @@ ItemLoader objects
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
- When instantiated with a `selector` or a `response` parameters
+ When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
data from web pages using :ref:`selectors `.
@@ -485,6 +477,8 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+ This is mostly used as a property so when attempting to override this
+ value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
@@ -494,7 +488,7 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
- the constructor.
+ the ``__init__`` method.
.. attribute:: default_input_processor
@@ -509,15 +503,15 @@ ItemLoader objects
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
- :class:`ItemLoader`, if only a response is given in the constructor.
- If a selector is given in the constructor this attribute is ignored.
+ :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+ If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
- It's either the selector given in the constructor or one created from
- the response given in the constructor using the
+ It's either the selector given in the ``__init__`` method or one created from
+ the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
@@ -642,7 +636,7 @@ Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
- values unchanged. It doesn't receive any constructor arguments, nor does it
+ values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example::
@@ -656,7 +650,7 @@ Here is a list of all built-in processors:
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
- It doesn't receive any constructor arguments, nor does it accept Loader contexts.
+ It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example::
@@ -667,7 +661,7 @@ Here is a list of all built-in processors:
.. class:: Join(separator=u' ')
- Returns the values joined with the separator given in the constructor, which
+ Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
@@ -705,7 +699,7 @@ Here is a list of all built-in processors:
those which do, this processor will pass the currently active :ref:`Loader
context ` through that parameter.
- The keyword arguments passed in the constructor are used as the default
+ The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
@@ -749,12 +743,12 @@ Here is a list of all built-in processors:
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
- constructor keyword arguments are used as default context values. See
+ ``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
- Queries the value using the json path provided to the constructor and returns the output.
+ Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 0986929ad..dd09477b8 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -193,6 +193,18 @@ to override some of the Scrapy settings regarding logging.
Module `logging.handlers `_
Further documentation on available handlers
+.. _custom-log-formats:
+
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending
+:class:`~scrapy.logformatter.LogFormatter` class and making
+:setting:`LOG_FORMATTER` point to your new class.
+
+.. autoclass:: scrapy.logformatter.LogFormatter
+ :members:
+
Advanced customization
----------------------
@@ -238,22 +250,23 @@ scrapy.utils.log module
.. autofunction:: configure_logging
- ``configure_logging`` is automatically called when using Scrapy commands,
- but needs to be called explicitly when running custom scripts. In that
- case, its usage is not required but it's recommended.
+ ``configure_logging`` is automatically called when using Scrapy commands
+ or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
+ when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
+ In that case, its usage is not required but it's recommended.
- If you plan on configuring the handlers yourself is still recommended you
- call this function, passing `install_root_handler=False`. Bear in mind
- there won't be any log output set by default in that case.
+ Another option when running custom scripts is to manually configure the logging.
+ To do this you can use `logging.basicConfig()`_ to set a basic root handler.
- To get you started on manually configuring logging's output, you can use
- `logging.basicConfig()`_ to set a basic root handler. This is an example
- on how to redirect ``INFO`` or higher messages to a file::
+ Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
+ so it is recommended to only use `logging.basicConfig()`_ together with
+ :class:`~scrapy.crawler.CrawlerRunner`.
+
+ This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
- configure_logging(install_root_handler=False)
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index c60b55391..206e7cfa5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -132,7 +132,7 @@ For example, the following image URL::
http://www.example.com/image.jpg
-Whose `SHA1 hash` is::
+Whose ``SHA1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
@@ -171,7 +171,7 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
-Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@@ -392,6 +392,36 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
+ .. method:: file_path(request, response, info)
+
+ This method is called once per downloaded item. It returns the
+ download path of the file originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request ` and
+ :class:`info `.
+
+ You can override this method to customize the download path of each file.
+
+ For example, if file URLs end like regular paths (e.g.
+ ``https://example.com/a/b/c/foo.png``), you can use the following
+ approach to download all files into the ``files`` folder with their
+ original filenames (e.g. ``files/foo.png``)::
+
+ import os
+ from urllib.parse import urlparse
+
+ from scrapy.pipelines.files import FilesPipeline
+
+ class MyFilesPipeline(FilesPipeline):
+
+ def file_path(self, request, response, info):
+ return 'files/' + os.path.basename(urlparse(request.url).path)
+
+ By default the :meth:`file_path` method returns
+ ``full/.``.
+
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@@ -411,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
* ``success`` is a boolean which is ``True`` if the image was downloaded
successfully or ``False`` if it failed for some reason
- * ``file_info_or_error`` is a dict containing the following keys (if success
- is ``True``) or a `Twisted Failure`_ if there was a problem.
+ * ``file_info_or_error`` is a dict containing the following keys (if
+ success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
+ there was a problem.
* ``url`` - the url where the file was downloaded from. This is the url of
the request returned from the :meth:`~get_media_requests`
@@ -475,6 +506,36 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
+ .. method:: file_path(request, response, info)
+
+ This method is called once per downloaded item. It returns the
+ download path of the file originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request ` and
+ :class:`info `.
+
+ You can override this method to customize the download path of each file.
+
+ For example, if file URLs end like regular paths (e.g.
+ ``https://example.com/a/b/c/foo.png``), you can use the following
+ approach to download all files into the ``files`` folder with their
+ original filenames (e.g. ``files/foo.png``)::
+
+ import os
+ from urllib.parse import urlparse
+
+ from scrapy.pipelines.images import ImagesPipeline
+
+ class MyImagesPipeline(ImagesPipeline):
+
+ def file_path(self, request, response, info):
+ return 'files/' + os.path.basename(urlparse(request.url).path)
+
+ By default the :meth:`file_path` method returns
+ ``full/.``.
+
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@@ -517,5 +578,4 @@ above::
item['image_paths'] = image_paths
return item
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 02cfa9b05..e3e8fdc72 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it.
# Your spider definition
...
- process = CrawlerProcess({
- 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+ process = CrawlerProcess(settings={
+ 'FEED_FORMAT': 'json',
+ 'FEED_URI': 'items.json'
})
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
-Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
-acquainted with its usage details.
+Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
+documentation to get acquainted with its usage details.
If you are inside a Scrapy project there are some additional helpers you can
use to import those components within the project. You can automatically import
@@ -80,7 +81,7 @@ returned by the :meth:`CrawlerRunner.crawl
` method.
Here's an example of its usage, along with a callback to manually stop the
-reactor after `MySpider` has finished running.
+reactor after ``MySpider`` has finished running.
::
@@ -100,7 +101,7 @@ reactor after `MySpider` has finished running.
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
-.. seealso:: `Twisted Reactor Overview`_.
+.. seealso:: :doc:`twisted:core/howto/reactor-basics`
.. _run-multiple-spiders:
@@ -252,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 76360b15f..4cf367d96 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@@ -50,7 +50,7 @@ Request objects
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
- ``str`` using the `encoding` passed (which defaults to ``utf-8``). If
+ ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
@@ -83,17 +83,21 @@ Request objects
.. reqmeta:: dont_merge_cookies
When some site returns cookies (in a response) those are stored in the
- cookies for that domain and will be sent again in future requests. That's
- the typical behaviour of any regular web browser. However, if, for some
- reason, you want to avoid merging with existing cookies you can instruct
- Scrapy to do so by setting the ``dont_merge_cookies`` key to True in the
- :attr:`Request.meta`.
+ cookies for that domain and will be sent again in future requests.
+ That's the typical behaviour of any regular web browser.
- Example of request without merging cookies::
+ To create a request that does not send stored cookies and does not
+ store received cookies, set the ``dont_merge_cookies`` key to ``True``
+ in :attr:`request.meta `.
- request_with_cookies = Request(url="http://www.example.com",
- cookies={'currency': 'USD', 'country': 'UY'},
- meta={'dont_merge_cookies': True})
+ Example of a request that sends manually-defined cookies and ignores
+ cookie storage::
+
+ Request(
+ url="http://www.example.com",
+ cookies={'currency': 'USD', 'country': 'UY'},
+ meta={'dont_merge_cookies': True},
+ )
For more info see :ref:`cookies-mw`.
:type cookies: dict or list
@@ -117,8 +121,8 @@ Request objects
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
- with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
- as first parameter.
+ with 404 HTTP errors and such. It receives a
+ :exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
:type errback: callable
@@ -126,11 +130,14 @@ Request objects
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
+ :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+ :type cb_kwargs: dict
+
.. attribute:: Request.url
A string containing the URL of this request. Keep in mind that this
attribute contains the escaped URL, so it can differ from the URL passed in
- the constructor.
+ the ``__init__`` method.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@@ -165,6 +172,17 @@ Request objects
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
+ .. attribute:: Request.cb_kwargs
+
+ A dictionary that contains arbitrary metadata for this request. Its contents
+ will be passed to the Request's callback as keyword arguments. It is empty
+ for new Requests, which means by default callbacks only get a :class:`Response`
+ object as argument.
+
+ This dict is `shallow copied`_ when the request is cloned using the
+ ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+ spider, from the ``response.cb_kwargs`` attribute.
+
.. _shallow copied: https://docs.python.org/2/library/copy.html
.. method:: Request.copy()
@@ -172,14 +190,16 @@ Request objects
Return a new Request which is a copy of this Request. See also:
:ref:`topics-request-response-ref-request-callback-arguments`.
- .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+ .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
Return a Request object with the same members, except for those members
given new values by whichever keyword arguments are specified. The
- attribute :attr:`Request.meta` is copied by default (unless a new value
- is given in the ``meta`` argument). See also
+ :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+ copied by default (unless new values are given as arguments). See also
:ref:`topics-request-response-ref-request-callback-arguments`.
+ .. automethod:: from_curl
+
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@@ -200,25 +220,31 @@ Example::
self.logger.info("Visited %s", response.url)
In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
+functions so you can receive the arguments later, in the second callback.
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+::
- def parse_page1(self, response):
- item = MyItem()
- item['main_url'] = response.url
- request = scrapy.Request("http://www.example.com/some_page.html",
- callback=self.parse_page2)
- request.meta['item'] = item
+ def parse(self, response):
+ request = scrapy.Request('http://www.example.com/index.html',
+ callback=self.parse_page2,
+ cb_kwargs=dict(main_url=response.url))
+ request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
yield request
- def parse_page2(self, response):
- item = response.meta['item']
- item['other_url'] = response.url
- yield item
+ def parse_page2(self, response, main_url, foo):
+ yield dict(
+ main_url=main_url,
+ other_url=response.url,
+ foo=foo,
+ )
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+ Prior to that, using :attr:`Request.meta` was recommended for passing
+ information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+ became the preferred way for handling user information, leaving :attr:`Request.meta`
+ for communication with components like middlewares and extensions.
.. _topics-request-response-ref-errbacks:
@@ -228,8 +254,8 @@ Using errbacks to catch exceptions in request processing
The errback of a request is a function that will be called when an exception
is raise while processing it.
-It receives a `Twisted Failure`_ instance as first parameter and can be
-used to track connection establishment timeouts, DNS errors etc.
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
@@ -299,6 +325,7 @@ Those are:
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
@@ -373,7 +400,7 @@ fields with form data from :class:`Response` objects.
.. class:: FormRequest(url, [formdata, ...])
- The :class:`FormRequest` class adds a new argument to the constructor. The
+ The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@@ -446,7 +473,7 @@ fields with form data from :class:`Response` objects.
:type dont_click: boolean
The other parameters of this class method are passed directly to the
- :class:`FormRequest` constructor.
+ :class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
@@ -512,11 +539,49 @@ method for this job. Here's an example spider which uses it::
# continue scraping with authenticated session...
+JsonRequest
+-----------
+
+The JsonRequest class extends the base :class:`Request` class with functionality for
+dealing with JSON requests.
+
+.. class:: JsonRequest(url, [... data, dumps_kwargs])
+
+ The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
+ remaining arguments are the same as for the :class:`Request` class and are
+ not documented here.
+
+ Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json``
+ and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
+
+ :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
+ if :attr:`Request.body` argument is provided this parameter will be ignored.
+ if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
+ set to ``'POST'`` automatically.
+ :type data: JSON serializable object
+
+ :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
+ data into JSON format.
+ :type dumps_kwargs: dict
+
+.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
+
+JsonRequest usage example
+-------------------------
+
+Sending a JSON POST request with a JSON payload::
+
+ data = {
+ 'name1': 'value1',
+ 'name2': 'value2',
+ }
+ yield JsonRequest(url='http://www.example.com/post/action', data=data)
+
Response objects
================
-.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
+.. autoclass:: Response
A :class:`Response` object represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -531,8 +596,8 @@ Response objects
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
- :param body: the response body. To access the decoded text as str (unicode
- in Python 2) you can use ``response.text`` from an encoding-aware
+ :param body: the response body. To access the decoded text as str you can use
+ ``response.text`` from an encoding-aware
:ref:`Response subclass `,
such as :class:`TextResponse`.
:type body: bytes
@@ -610,7 +675,7 @@ Response objects
.. attribute:: Response.flags
A list that contains flags for this response. Flags are labels used for
- tagging Responses. For example: `'cached'`, `'redirected`', etc. And
+ tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
they're shown on the string representation of the Response (`__str__`
method) which is used by the engine for logging.
@@ -656,7 +721,7 @@ TextResponse objects
:class:`Response` class, which is meant to be used only for binary data,
such as images, sounds or any media file.
- :class:`TextResponse` objects support a new constructor argument, in
+ :class:`TextResponse` objects support a new ``__init__`` method argument, in
addition to the base :class:`Response` objects. The remaining functionality
is the same as for the :class:`Response` class and is not documented here.
@@ -682,7 +747,7 @@ TextResponse objects
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
- (typically `ascii`) instead of the response encoding.
+ (typically ``ascii``) instead of the response encoding.
.. attribute:: TextResponse.encoding
@@ -690,7 +755,7 @@ TextResponse objects
A string with the encoding of this response. The encoding is resolved by
trying the following mechanisms, in order:
- 1. the encoding passed in the constructor `encoding` argument
+ 1. the encoding passed in the ``__init__`` method ``encoding`` argument
2. the encoding declared in the Content-Type HTTP header. If this
encoding is not valid (ie. unknown), it is ignored and the next
@@ -728,7 +793,7 @@ TextResponse objects
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
- kept for backwards compatibility; please prefer ``response.text``.
+ kept for backward compatibility; please prefer ``response.text``.
HtmlResponse objects
@@ -751,5 +816,4 @@ XmlResponse objects
adds encoding auto-discovering support by looking into the XML declaration
line. See :attr:`TextResponse.encoding`.
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index df1d67ae8..282a585d4 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -96,7 +96,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
Using selectors
---------------
-To explain how to use the selectors we'll use the `Scrapy shell` (which
+To explain how to use the selectors we'll use the ``Scrapy shell`` (which
provides interactive testing) and an example page located in the Scrapy
documentation server:
@@ -436,7 +436,7 @@ The following examples show how these methods map to each other.
>>> response.css('a::attr(href)').extract()
['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
+3. ``Selector.get()`` is the same as ``Selector.extract()``::
>>> response.css('a::attr(href)')[0].get()
'image1.html'
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 0ac26a9bd..a1d15a760 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -30,6 +30,8 @@ Python `import search path`_.
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
+.. _populating-settings:
+
Populating the settings
=======================
@@ -186,7 +188,6 @@ AWS_ENDPOINT_URL
Default: ``None``
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
-Only supported with ``botocore`` library.
.. setting:: AWS_USE_SSL
@@ -197,7 +198,6 @@ Default: ``None``
Use this option if you want to disable SSL connection for communication with
S3 or S3-like storage. By default SSL will be used.
-Only supported with ``botocore`` library.
.. setting:: AWS_VERIFY
@@ -207,7 +207,7 @@ AWS_VERIFY
Default: ``None``
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
-SSL verification will occur. Only supported with ``botocore`` library.
+SSL verification will occur.
.. setting:: AWS_REGION_NAME
@@ -217,7 +217,6 @@ AWS_REGION_NAME
Default: ``None``
The name of the region associated with the AWS client.
-Only supported with ``botocore`` library.
.. setting:: BOT_NAME
@@ -227,8 +226,7 @@ BOT_NAME
Default: ``'scrapybot'``
The name of the bot implemented by this Scrapy project (also known as the
-project name). This will be used to construct the User-Agent by default, and
-also for logging.
+project name). This name will be used for the logging too.
It's automatically populated with your project name when you create your
project with the :command:`startproject` command.
@@ -331,16 +329,16 @@ Default: ``0``
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
-An integer that is used to adjust the request priority based on its depth:
+An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
+a :class:`~scrapy.http.Request` based on its depth.
-- if zero (default), no priority adjustment is made from depth
-- **a positive value will decrease the priority, i.e. higher depth
- requests will be processed later** ; this is commonly used when doing
- breadth-first crawls (BFO)
-- a negative value will increase priority, i.e., higher depth requests
- will be processed sooner (DFO)
+The priority of a request is adjusted as follows::
-See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
+ request.priority = request.priority - ( depth * DEPTH_PRIORITY )
+
+As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
+priority (BFO), while negative values increase request priority (DFO). See
+also :ref:`faq-bfo-dfo`.
.. note::
@@ -438,9 +436,29 @@ or even enable client-side authentication (and various other things).
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
-If you do use a custom ContextFactory, make sure it accepts a ``method``
-parameter at init (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+If you do use a custom ContextFactory, make sure its ``__init__`` method
+accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging``
+parameter (``bool``) and a ``tls_ciphers`` parameter (see
+:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`).
+
+.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS
+
+DOWNLOADER_CLIENT_TLS_CIPHERS
+-----------------------------
+
+Default: ``'DEFAULT'``
+
+Use this setting to customize the TLS/SSL ciphers used by the default
+HTTP/1.1 downloader.
+
+The setting should contain a string in the `OpenSSL cipher list format`_,
+these ciphers will be used as client ciphers. Changing this setting may be
+necessary to access certain HTTPS websites: for example, you may need to use
+``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
+specific cipher that is not included in ``DEFAULT`` if a website requires it.
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@@ -468,6 +486,20 @@ This setting must be one of these string values:
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
+.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+-------------------------------------
+
+Default: ``False``
+
+Setting this to ``True`` will enable DEBUG level messages about TLS connection
+parameters after establishing HTTPS connections. The kind of information logged
+depends on the versions of OpenSSL and pyOpenSSL.
+
+This setting is only used for the default
+:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
.. setting:: DOWNLOADER_MIDDLEWARES
DOWNLOADER_MIDDLEWARES
@@ -538,6 +570,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti
When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
per ip address instead of per domain.
+.. _spider-download_delay-attribute:
+
You can also change this setting per spider by setting ``download_delay``
spider attribute.
@@ -599,7 +633,7 @@ The amount of time (in secs) that the downloader will wait before timing out.
DOWNLOAD_MAXSIZE
----------------
-Default: `1073741824` (1024MB)
+Default: ``1073741824`` (1024MB)
The maximum response size (in bytes) that downloader will download.
@@ -620,7 +654,7 @@ If you want to disable it set to 0.
DOWNLOAD_WARNSIZE
-----------------
-Default: `33554432` (32MB)
+Default: ``33554432`` (32MB)
The response size (in bytes) that downloader will start to warn.
@@ -758,6 +792,7 @@ Default: ``True``
Whether or not to use passive mode when initiating FTP transfers.
+.. reqmeta:: ftp_password
.. setting:: FTP_PASSWORD
FTP_PASSWORD
@@ -776,6 +811,7 @@ in ``Request`` meta.
.. _RFC 1635: https://tools.ietf.org/html/rfc1635
+.. reqmeta:: ftp_user
.. setting:: FTP_USER
FTP_USER
@@ -866,6 +902,15 @@ directives.
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: :class:`scrapy.logformatter.LogFormatter`
+
+The class to use for :ref:`formatting log messages ` for different actions.
+
.. setting:: LOG_LEVEL
LOG_LEVEL
@@ -897,6 +942,16 @@ Default: ``False``
If ``True``, the logs will just contain the root path. If it is set to ``False``
then it displays the component responsible for the log output
+.. setting:: LOGSTATS_INTERVAL
+
+LOGSTATS_INTERVAL
+-----------------
+
+Default: ``60.0``
+
+The interval (in seconds) between each logging printout of the stats
+by :class:`~scrapy.extensions.logstats.LogStats`.
+
.. setting:: MEMDEBUG_ENABLED
MEMDEBUG_ENABLED
@@ -1103,6 +1158,28 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
this option is enabled by default in settings.py file generated
by ``scrapy startproject`` command.
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.ProtegoRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
.. setting:: SCHEDULER
SCHEDULER
@@ -1155,9 +1232,14 @@ Type of in-memory queue used by scheduler. Other available type is:
SCHEDULER_PRIORITY_QUEUE
------------------------
-Default: ``'queuelib.PriorityQueue'``
+Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
-Type of priority queue used by scheduler.
+Type of priority queue used by the scheduler. Another available type is
+``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
+``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
+domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
.. setting:: SPIDER_CONTRACTS
@@ -1356,7 +1438,10 @@ USER_AGENT
Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
Settings documented elsewhere:
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ff07b9d55..3f29aa323 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
Deferred signal handlers
========================
-Some signals support returning `Twisted deferreds`_ from their handlers, see
-the :ref:`topics-signals-ref` below to know which ones.
+Some signals support returning :class:`~twisted.internet.defer.Deferred`
+objects from their handlers, see the :ref:`topics-signals-ref` below to know
+which ones.
-.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
.. _topics-signals-ref:
@@ -155,8 +155,8 @@ item_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
- :param failure: the exception raised as a Twisted `Failure`_ object
- :type failure: `Failure`_ object
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
spider_closed
-------------
@@ -236,8 +236,8 @@ spider_error
This signal does not support returning deferreds from their handlers.
- :param failure: the exception raised as a Twisted `Failure`_ object
- :type failure: `Failure`_ object
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@@ -333,5 +333,3 @@ response_downloaded
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
-
-.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2b7e42771..0e8210130 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
-in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
SPIDER_MIDDLEWARES = {
@@ -54,11 +54,17 @@ value. For example, if you want to disable the off-site middleware::
Finally, keep in mind that some middlewares may need to be enabled through a
particular setting. See each middleware documentation for more info.
+.. _custom-spider-middleware:
+
Writing your own spider middleware
==================================
-Each middleware component is a Python class that defines one or more of the
-following methods:
+Each spider middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings `.
.. module:: scrapy.spidermiddlewares
@@ -78,7 +84,8 @@ following methods:
If it raises an exception, Scrapy won't bother calling any other spider
middleware :meth:`process_spider_input` and will call the request
- errback. The output of the errback is chained back in the other
+ errback if there is one, otherwise it will start the :meth:`process_spider_exception`
+ chain. The output of the errback is chained back in the other
direction for :meth:`process_spider_output` to process it, or
:meth:`process_spider_exception` if it raised an exception.
@@ -112,8 +119,8 @@ following methods:
.. method:: process_spider_exception(response, exception, spider)
- This method is called when a spider or :meth:`process_spider_input`
- method (from other spider middleware) raises an exception.
+ This method is called when a spider or :meth:`process_spider_output`
+ method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Request`, dict or
@@ -125,7 +132,8 @@ following methods:
exception reaches the engine (where it's logged and discarded).
If it returns an iterable the :meth:`process_spider_output` pipeline
- kicks in, and no other :meth:`process_spider_exception` will be called.
+ kicks in, starting from the next spider middleware, and no other
+ :meth:`process_spider_exception` will be called.
:param response: the response being processed when the exception was
raised
@@ -200,7 +208,7 @@ DepthMiddleware
.. class:: DepthMiddleware
DepthMiddleware is used for tracking the depth of each Request inside the
- site being scraped. It works by setting `request.meta['depth'] = 0` whenever
+ site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
there is no value previously set (usually just the first Request) and
incrementing it by 1 otherwise.
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 742a88659..d65a43afd 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -72,8 +72,6 @@ scrapy.Spider
spider that crawls ``mywebsite.com`` would often be called
``mywebsite``.
- .. note:: In Python 2 this must be ASCII only.
-
.. attribute:: allowed_domains
An optional list of strings containing domains that this spider is
@@ -129,7 +127,7 @@ scrapy.Spider
You probably won't need to override this directly because the default
implementation acts as a proxy to the :meth:`__init__` method, calling
- it with the given arguments `args` and named arguments `kwargs`.
+ it with the given arguments ``args`` and named arguments ``kwargs``.
Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
attributes in the new instance so they can be accessed later inside the
@@ -190,7 +188,7 @@ scrapy.Spider
.. method:: log(message, [level, component])
Wrapper that sends a log message through the Spider's :attr:`logger`,
- kept for backwards compatibility. For more information see
+ kept for backward compatibility. For more information see
:ref:`topics-logging-from-spiders`.
.. method:: closed(reason)
@@ -298,13 +296,13 @@ The above example can also be written as follows::
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
-If you were to set the `start_urls` attribute from the command line,
+If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval `_
or `json.loads `_
and then set it as an attribute.
-Otherwise, you would cause iteration over a `start_urls` string
+Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
resulting in each character being seen as a separate url.
@@ -374,16 +372,23 @@ CrawlSpider
Crawling rules
~~~~~~~~~~~~~~
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
+.. autoclass:: Rule
``link_extractor`` is a :ref:`Link Extractor ` object which
- defines how links will be extracted from each crawled page.
+ defines how links will be extracted from each crawled page. Each produced link will
+ be used to generate a :class:`~scrapy.http.Request` object, which will contain the
+ link's text in its ``meta`` dictionary (under the ``link_text`` key).
+ If omitted, a default link extractor created with no arguments will be used,
+ resulting in all links being extracted.
``callback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called for each link extracted with
- the specified link_extractor. This callback receives a response as its first
- argument and must return a list containing :class:`~scrapy.item.Item` and/or
- :class:`~scrapy.http.Request` objects (or any subclass of them).
+ the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
+ as its first argument and must return either a single instance or an iterable of
+ :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+ (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
+ object will contain the text of the link that produced the :class:`~scrapy.http.Request`
+ in its ``meta`` dictionary (under the ``link_text`` key)
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
callback, since the :class:`CrawlSpider` uses the ``parse`` method
@@ -402,10 +407,12 @@ Crawling rules
of links extracted from each response using the specified ``link_extractor``.
This is mainly used for filtering purposes.
- ``process_request`` is a callable, or a string (in which case a method from
- the spider object with that name will be used) which will be called with
- every request extracted by this rule, and must return a request or None (to
- filter out the request).
+ ``process_request`` is a callable (or a string, in which case a method from
+ the spider object with that name will be used) which will be called for every
+ :class:`~scrapy.http.Request` extracted by this rule. This callable should
+ take said request as first argument and the :class:`~scrapy.http.Response`
+ from which the request originated as second argument. It must return a
+ ``Request`` object or ``None`` (to filter out the request).
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
@@ -436,6 +443,7 @@ Let's now take a look at an example CrawlSpider with rules::
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
+ item['link_text'] = response.meta['link_text']
return item
@@ -655,7 +663,7 @@ SitemapSpider
.. attribute:: sitemap_follow
- A list of regexes of sitemap that should be followed. This is is only
+ A list of regexes of sitemap that should be followed. This is only
for sites that use `Sitemap index files`_ that point to other sitemap
files.
@@ -682,7 +690,7 @@ SitemapSpider
.. method:: sitemap_filter(entries)
- This is a filter funtion that could be overridden to select sitemap entries
+ This is a filter function that could be overridden to select sitemap entries
based on their attributes.
For example::
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index dd0c6216b..38648ec55 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
default Stats Collector used is the :class:`MemoryStatsCollector`.
-.. module:: scrapy.statscollectors
- :synopsis: Stats Collectors
+.. currentmodule:: scrapy.statscollectors
MemoryStatsCollector
--------------------
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index bf2ffa443..7db7e4f6b 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -1,12 +1,11 @@
+.. currentmodule:: scrapy.extensions.telnet
+
.. _topics-telnetconsole:
==============
Telnet Console
==============
-.. module:: scrapy.extensions.telnet
- :synopsis: The Telnet Console
-
Scrapy comes with a built-in telnet console for inspecting and controlling a
Scrapy running process. The telnet console is just a regular python shell
running inside the Scrapy process, so you can do literally anything from it.
@@ -45,7 +44,7 @@ the console you need to type::
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
-autogenerated Password can be seen on scrapy logs like the example bellow::
+autogenerated Password can be seen on scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
deleted file mode 100644
index 81ce800aa..000000000
--- a/docs/topics/ubuntu.rst
+++ /dev/null
@@ -1,41 +0,0 @@
-:orphan: Ubuntu packages are obsolete
-
-.. _topics-ubuntu:
-
-===============
-Ubuntu packages
-===============
-
-.. versionadded:: 0.10
-
-`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
-those in Ubuntu, and more stable too since they're continuously built from
-`GitHub repo`_ (master & stable branches) and so they contain the latest bug
-fixes.
-
-.. caution:: These packages are currently not updated and may not work on
- Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
-
-To use the packages:
-
-1. Import the GPG key used to sign Scrapy packages into APT keyring::
-
- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
-
-2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command::
-
- echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
-
-3. Update package lists and install the scrapy package:
-
- .. parsed-literal::
-
- sudo apt-get update && sudo apt-get install scrapy
-
-.. note:: Repeat step 3 if you are trying to upgrade Scrapy.
-
-.. warning:: `python-scrapy` is a different package provided by official debian
- repositories, it's very outdated and it isn't supported by Scrapy team.
-
-.. _Scrapinghub: https://scrapinghub.com/
-.. _GitHub repo: https://github.com/scrapy/scrapy
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 0421ba544..227085f02 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
* *A* is the major version. This will rarely change and will signify very
large changes.
* *B* is the release number. This will include many changes including features
- and things that possibly break backwards compatibility, although we strive to
+ and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
* *C* is the bugfix release number.
diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 564991aa8..e995947cb 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -1,25 +1,210 @@
#compdef scrapy
-
-# zsh completion for the Scrapy command-line tool
-
_scrapy() {
- local curcontext="$curcontext" cmd spiders
+ local context state state_descr line
typeset -A opt_args
- cmd=$words[2]
-
- case "$cmd" in
- crawl|edit|check)
- spiders=$(scrapy list 2>/dev/null) || spiders=""
- if [[ -n "$spiders" ]]; then
- compadd `echo $spiders`
- fi
- ;;
- *)
- if [[ CURRENT -eq 2 ]]; then
- _arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
- fi
- ;;
+ _arguments \
+ "(- 1 *)--help[Help]" \
+ "1: :->command" \
+ "*:: :->args"
+
+ case $state in
+ command)
+ _scrapy_cmds
+ ;;
+ args)
+ case $words[1] in
+ bench)
+ _scrapy_glb_opts
+ ;;
+ fetch)
+ local options=(
+ '--headers[print response HTTP headers instead of body]'
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '1::URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ genspider)
+ local options=(
+ {-l,--list}'[List available templates]'
+ {-e,--edit}'[Edit spider after creating it]'
+ '--force[If the spider already exists, overwrite it with the template]'
+ {-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
+ {-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
+ '1:name:(NAME)'
+ '2:domain:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ runspider)
+ local options=(
+ {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+ {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '1:spider file:_files -g \*.py'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ settings)
+ local options=(
+ '--get=[print raw setting value]:option:(SETTING)'
+ '--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
+ '--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
+ '--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
+ '--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ shell)
+ local options=(
+ '-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '::file:_files -g \*.html'
+ '::URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ startproject)
+ local options=(
+ '1:name:(NAME)'
+ '2:dir:_dir_list'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ version)
+ local options=(
+ {-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ view)
+ local options=(
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '1:URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ check)
+ local options=(
+ '(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
+ {-v,--verbose}'[print contract tests for all spiders]'
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ crawl)
+ local options=(
+ {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+ {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ edit)
+ local options=(
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ list)
+ _scrapy_glb_opts
+ ;;
+ parse)
+ local options=(
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '--spider[use this spider without looking for one]:spider:_scrapy_spiders'
+ '--pipelines[process items through pipelines]'
+ "--nolinks[don't show links to follow (extracted requests)]"
+ "--noitems[don't show scraped items]"
+ '--nocolour[avoid using pygments to colorize the output]'
+ {-r,--rules}'[use CrawlSpider rules to discover the callback]'
+ {-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
+ {-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
+ '--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
+ {-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
+ {-v,--verbose}'[print each depth level one by one]'
+ '1:URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ esac
+ ;;
esac
}
-_scrapy
\ No newline at end of file
+_scrapy_cmds() {
+ local -a commands project_commands
+ commands=(
+ 'bench:Run quick benchmark test'
+ 'fetch:Fetch a URL using the Scrapy downloader'
+ 'genspider:Generate new spider using pre-defined templates'
+ 'runspider:Run a self-contained spider (without creating a project)'
+ 'settings:Get settings values'
+ 'shell:Interactive scraping console'
+ 'startproject:Create new project'
+ 'version:Print Scrapy version'
+ 'view:Open URL in browser, as seen by Scrapy'
+ )
+ project_commands=(
+ 'check:Check spider contracts'
+ 'crawl:Run a spider'
+ 'edit:Edit spider'
+ 'list:List available spiders'
+ 'parse:Parse URL (using its spider) and print the results'
+ )
+ if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+ commands=(${commands[@]} ${project_commands[@]})
+ fi
+ _describe -t common-commands 'common commands' commands
+}
+
+_scrapy_glb_opts() {
+ local -a options
+ options=(
+ '(- *)'{-h,--help}'[show this help message and exit]'
+ '(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
+ '--pidfile=[write process ID to FILE]:file:_files'
+ '--profile=[write python cProfile stats to FILE]:file:_files'
+ '(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
+ '(-L --loglevel --logfile)--nolog[disable logging completely]'
+ '--pdb[enable pdb on failure]'
+ '*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
+ )
+ options=(${options[@]} "$@")
+ _arguments $options
+}
+
+_httpie_urls() {
+
+ local ret=1
+
+ if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
+ local expl
+ compset -S '[^:/]*' && compstate[to_end]=''
+ _wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
+ else
+ _urls && ret=0
+ fi
+
+ return $ret
+
+}
+
+_scrapy_spiders() {
+
+ local ret=1
+
+ if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+ compadd -S '' $(scrapy list) && ret=0
+ else
+ compadd -S '' SPIDER && ret=0
+ fi
+
+ return $ret
+}
+
+_scrapy $@
diff --git a/pytest.ini b/pytest.ini
index 73d169601..33c34b8e8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,5 +2,249 @@
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+ --assert=plain
+ --doctest-modules
+ --ignore=docs/_ext
+ --ignore=docs/conf.py
+ --ignore=docs/news.rst
+ --ignore=docs/topics/developer-tools.rst
+ --ignore=docs/topics/dynamic-content.rst
+ --ignore=docs/topics/items.rst
+ --ignore=docs/topics/leaks.rst
+ --ignore=docs/topics/loaders.rst
+ --ignore=docs/topics/selectors.rst
+ --ignore=docs/topics/shell.rst
+ --ignore=docs/topics/stats.rst
+ --ignore=docs/topics/telnetconsole.rst
+ --ignore=docs/utils
twisted = 1
+flake8-ignore =
+ # Files that are only meant to provide top-level imports are expected not
+ # to use any of their imports:
+ scrapy/core/downloader/handlers/http.py F401
+ scrapy/http/__init__.py F401
+ # Issues pending a review:
+ # extras
+ extras/qps-bench-server.py E261 E501
+ extras/qpsclient.py E501 E261 E501
+ # scrapy/commands
+ scrapy/commands/__init__.py E128 E501
+ scrapy/commands/check.py E501
+ scrapy/commands/crawl.py E501
+ scrapy/commands/edit.py E501
+ scrapy/commands/fetch.py E401 E501 E128 E502 E731
+ scrapy/commands/genspider.py E128 E501 E502
+ scrapy/commands/parse.py E128 E501 E731 E226
+ scrapy/commands/runspider.py E501
+ scrapy/commands/settings.py E128
+ scrapy/commands/shell.py E128 E501 E502
+ scrapy/commands/startproject.py E502 E127 E501 E128
+ scrapy/commands/version.py E501 E128
+ # scrapy/contracts
+ scrapy/contracts/__init__.py E501 W504
+ scrapy/contracts/default.py E502 E128
+ # scrapy/core
+ scrapy/core/engine.py E261 E501 E128 E127 E306 E502
+ scrapy/core/scheduler.py E501
+ scrapy/core/scraper.py E501 E306 E261 E128 W504
+ scrapy/core/spidermw.py E501 E731 E502 E126 E226
+ scrapy/core/downloader/__init__.py E501
+ scrapy/core/downloader/contextfactory.py E501 E128 E126
+ scrapy/core/downloader/middleware.py E501 E502
+ scrapy/core/downloader/tls.py E501 E305 E241
+ scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
+ scrapy/core/downloader/handlers/__init__.py E501
+ scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
+ scrapy/core/downloader/handlers/http10.py E501
+ scrapy/core/downloader/handlers/http11.py E501
+ scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
+ # scrapy/downloadermiddlewares
+ scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
+ scrapy/downloadermiddlewares/decompression.py E501
+ scrapy/downloadermiddlewares/defaultheaders.py E501
+ scrapy/downloadermiddlewares/httpcache.py E501 E126
+ scrapy/downloadermiddlewares/httpcompression.py E502 E128
+ scrapy/downloadermiddlewares/httpproxy.py E501
+ scrapy/downloadermiddlewares/redirect.py E501 W504
+ scrapy/downloadermiddlewares/retry.py E501 E126
+ scrapy/downloadermiddlewares/robotstxt.py E501
+ scrapy/downloadermiddlewares/stats.py E501
+ # scrapy/extensions
+ scrapy/extensions/closespider.py E501 E502 E128 E123
+ scrapy/extensions/corestats.py E501
+ scrapy/extensions/feedexport.py E128 E501
+ scrapy/extensions/httpcache.py E128 E501 E303
+ scrapy/extensions/memdebug.py E501
+ scrapy/extensions/spiderstate.py E501
+ scrapy/extensions/telnet.py E501 W504
+ scrapy/extensions/throttle.py E501
+ # scrapy/http
+ scrapy/http/common.py E501
+ scrapy/http/cookies.py E501
+ scrapy/http/request/__init__.py E501
+ scrapy/http/request/form.py E501 E123
+ scrapy/http/request/json_request.py E501
+ scrapy/http/response/__init__.py E501 E128 W293 W291
+ scrapy/http/response/text.py E501 W293 E128 E124
+ # scrapy/linkextractors
+ scrapy/linkextractors/__init__.py E731 E502 E501 E402
+ scrapy/linkextractors/lxmlhtml.py E501 E731 E226
+ # scrapy/loader
+ scrapy/loader/__init__.py E501 E502 E128
+ scrapy/loader/processors.py E501
+ # scrapy/pipelines
+ scrapy/pipelines/files.py E116 E501 E266
+ scrapy/pipelines/images.py E265 E501
+ scrapy/pipelines/media.py E125 E501 E266
+ # scrapy/selector
+ scrapy/selector/__init__.py F403
+ scrapy/selector/unified.py E501 E111
+ # scrapy/settings
+ scrapy/settings/__init__.py E501
+ scrapy/settings/default_settings.py E501 E261 E114 E116 E226
+ scrapy/settings/deprecated.py E501
+ # scrapy/spidermiddlewares
+ scrapy/spidermiddlewares/httperror.py E501
+ scrapy/spidermiddlewares/offsite.py E501
+ scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
+ scrapy/spidermiddlewares/urllength.py E501
+ # scrapy/spiders
+ scrapy/spiders/__init__.py E501 E402
+ scrapy/spiders/crawl.py E501
+ scrapy/spiders/feed.py E501 E261
+ scrapy/spiders/sitemap.py E501
+ # scrapy/utils
+ scrapy/utils/benchserver.py E501
+ scrapy/utils/conf.py E402 E502 E501
+ scrapy/utils/console.py E261 E306 E305
+ scrapy/utils/datatypes.py E501 E226
+ scrapy/utils/decorators.py E501
+ scrapy/utils/defer.py E501 E128
+ scrapy/utils/deprecate.py E128 E501 E127 E502
+ scrapy/utils/engine.py E261
+ scrapy/utils/gz.py E305 E501 W504
+ scrapy/utils/http.py F403 E226
+ scrapy/utils/httpobj.py E501
+ scrapy/utils/iterators.py E501 E701
+ scrapy/utils/log.py E128 W503
+ scrapy/utils/markup.py F403 W292
+ scrapy/utils/misc.py E501 E226
+ scrapy/utils/multipart.py F403 W292
+ scrapy/utils/project.py E501
+ scrapy/utils/python.py E501
+ scrapy/utils/reactor.py E226
+ scrapy/utils/reqser.py E501
+ scrapy/utils/request.py E127 E501
+ scrapy/utils/response.py E501 E128
+ scrapy/utils/signal.py E501 E128
+ scrapy/utils/sitemap.py E501
+ scrapy/utils/spider.py E271 E501
+ scrapy/utils/ssl.py E501
+ scrapy/utils/test.py E501
+ scrapy/utils/url.py E501 F403 E128 F405
+ # scrapy
+ scrapy/__init__.py E402 E501
+ scrapy/_monkeypatches.py W293
+ scrapy/cmdline.py E502 E501
+ scrapy/crawler.py E501
+ scrapy/dupefilters.py E501 E202
+ scrapy/exceptions.py E501
+ scrapy/exporters.py E501 E261 E226
+ scrapy/interfaces.py E501
+ scrapy/item.py E501 E128
+ scrapy/link.py E501
+ scrapy/logformatter.py E501 W293
+ scrapy/mail.py E402 E128 E501 E502
+ scrapy/middleware.py E502 E128 E501
+ scrapy/pqueues.py E501
+ scrapy/responsetypes.py E128 E501 E305
+ scrapy/robotstxt.py E501
+ scrapy/shell.py E501
+ scrapy/signalmanager.py E501
+ scrapy/spiderloader.py E225 F841 E501 E126
+ scrapy/squeues.py E128
+ scrapy/statscollectors.py E501
+ # tests
+ tests/__init__.py E402 E501
+ tests/mockserver.py E401 E501 E126 E123
+ tests/pipelines.py F841 E226
+ tests/spiders.py E501 E127
+ tests/test_closespider.py E501 E127
+ tests/test_command_fetch.py E501 E261
+ tests/test_command_parse.py E501 E128 E303 E226
+ tests/test_command_shell.py E501 E128
+ tests/test_commands.py E128 E501
+ tests/test_contracts.py E501 E128 W293
+ tests/test_crawl.py E501 E741 E265
+ tests/test_crawler.py F841 E306 E501
+ tests/test_dependencies.py F841 E501 E305
+ tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
+ tests/test_downloadermiddleware.py E501
+ tests/test_downloadermiddleware_ajaxcrawlable.py E501
+ tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
+ tests/test_downloadermiddleware_decompression.py E127
+ tests/test_downloadermiddleware_defaultheaders.py E501
+ tests/test_downloadermiddleware_downloadtimeout.py E501
+ tests/test_downloadermiddleware_httpcache.py E501 E305
+ tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
+ tests/test_downloadermiddleware_httpproxy.py E501 E128
+ tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
+ tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
+ tests/test_downloadermiddleware_robotstxt.py E501
+ tests/test_downloadermiddleware_stats.py E501
+ tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
+ tests/test_engine.py E401 E501 E502 E128 E261
+ tests/test_exporters.py E501 E731 E306 E128 E124
+ tests/test_extension_telnet.py F841
+ tests/test_feedexport.py E501 F841 E241
+ tests/test_http_cookies.py E501
+ tests/test_http_headers.py E501
+ tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
+ tests/test_http_response.py E501 E301 E502 E128 E265
+ tests/test_item.py E701 E128 F841 E306
+ tests/test_link.py E501
+ tests/test_linkextractors.py E501 E128 E124
+ tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
+ tests/test_logformatter.py E128 E501 E122
+ tests/test_mail.py E128 E501 E305
+ tests/test_middleware.py E501 E128
+ tests/test_pipeline_crawl.py E131 E501 E128 E126
+ tests/test_pipeline_files.py E501 W293 E303 E272 E226
+ tests/test_pipeline_images.py F841 E501 E303
+ tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+ tests/test_proxy_connect.py E501 E741
+ tests/test_request_cb_kwargs.py E501
+ tests/test_responsetypes.py E501 E305
+ tests/test_robotstxt_interface.py E501 W291 E501
+ tests/test_scheduler.py E501 E126 E123
+ tests/test_selector.py E501 E127
+ tests/test_spider.py E501
+ tests/test_spidermiddleware.py E501 E226
+ tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+ tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
+ tests/test_spidermiddleware_output_chain.py E501 W293 E226
+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
+ tests/test_squeues.py E501 E701 E741
+ tests/test_utils_conf.py E501 E303 E128
+ tests/test_utils_curl.py E501
+ tests/test_utils_datatypes.py E402 E501 E305
+ tests/test_utils_defer.py E306 E261 E501 F841 E226
+ tests/test_utils_deprecate.py F841 E306 E501
+ tests/test_utils_http.py E501 E502 E128 W504
+ tests/test_utils_iterators.py E501 E128 E129 E303 E241
+ tests/test_utils_log.py E741 E226
+ tests/test_utils_python.py E501 E303 E731 E701 E305
+ tests/test_utils_reqser.py E501 E128
+ tests/test_utils_request.py E501 E128 E305
+ tests/test_utils_response.py E501
+ tests/test_utils_signal.py E741 F841 E731 E226
+ tests/test_utils_sitemap.py E128 E501 E124
+ tests/test_utils_spider.py E261 E305
+ tests/test_utils_template.py E305
+ tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
+ tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
+ tests/test_cmdline/__init__.py E502 E501
+ tests/test_settings/__init__.py E501 E128
+ tests/test_spiderloader/__init__.py E128 E501
+ tests/test_utils_misc/__init__.py E501
diff --git a/requirements-py2.txt b/requirements-py2.txt
deleted file mode 100644
index 0771aae3a..000000000
--- a/requirements-py2.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
-PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 5a5d4c95a..2c98e6f6d 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,10 +1,18 @@
-Twisted>=17.9.0
-lxml>=3.2.4
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+Twisted>=17.9.0
+w3lib>=1.17.0
+protego>=0.1.15
+
+pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0 # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+zope.interface>=4.1.3
diff --git a/scrapy/VERSION b/scrapy/VERSION
index dc1e644a1..27f9cd322 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.6.0
+1.8.0
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 03ec6c667..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -14,8 +14,8 @@ del pkgutil
# Check minimum required Python version
import sys
-if sys.version_info < (2, 7):
- print("Scrapy %s requires Python 2.7" % __version__)
+if sys.version_info < (3, 5):
+ print("Scrapy %s requires Python 3.5" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index f55ecc213..1f8067b35 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,19 +1,5 @@
-import sys
from six.moves import copyreg
-if sys.version_info[0] == 2:
- from urlparse import urlparse
-
- # workaround for https://bugs.python.org/issue7904 - Python < 2.7
- if urlparse('s3://bucket/key').netloc != 'bucket':
- from urlparse import uses_netloc
- uses_netloc.append('s3')
-
- # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
- if urlparse('s3://bucket/key?key=value').query != 'key=value':
- from urlparse import uses_query
- uses_query.append('s3')
-
# Undo what Twisted's perspective broker adds to pickle register
# to prevent bugs like Twisted#7989 while serializing requests
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index dc6b59fe0..418dc1ac9 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,5 +1,6 @@
from __future__ import print_function
-import sys, os
+import sys
+import os
import optparse
import cProfile
import inspect
@@ -14,6 +15,7 @@ from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
+
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# scrapy.utils.spider.iter_spider_classes
@@ -25,6 +27,7 @@ def _iter_command_classes(module_name):
not obj == ScrapyCommand:
yield obj
+
def _get_commands_from_module(module, inproject):
d = {}
for cmd in _iter_command_classes(module):
@@ -33,6 +36,7 @@ def _get_commands_from_module(module, inproject):
d[cmdname] = cmd()
return d
+
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
@@ -43,6 +47,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
raise Exception("Invalid entry point %s" % entry_point.name)
return cmds
+
def _get_commands_dict(settings, inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds.update(_get_commands_from_entry_points(inproject))
@@ -51,6 +56,7 @@ def _get_commands_dict(settings, inproject):
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
+
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
@@ -59,13 +65,15 @@ def _pop_command_name(argv):
return arg
i += 1
+
def _print_header(settings, inproject):
if inproject:
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
- settings['BOT_NAME']))
+ settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
+
def _print_commands(settings, inproject):
_print_header(settings, inproject)
print("Usage:")
@@ -80,11 +88,13 @@ def _print_commands(settings, inproject):
print()
print('Use "scrapy -h" to see more info about a command')
+
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print('Use "scrapy" to see available commands')
+
def _run_print_help(parser, func, *a, **kw):
try:
func(*a, **kw)
@@ -95,41 +105,27 @@ def _run_print_help(parser, func, *a, **kw):
parser.print_help()
sys.exit(2)
+
def execute(argv=None, settings=None):
if argv is None:
argv = sys.argv
- # --- backwards compatibility for scrapy.conf.settings singleton ---
- if settings is None and 'scrapy.conf' in sys.modules:
- from scrapy import conf
- if hasattr(conf, 'settings'):
- settings = conf.settings
- # ------------------------------------------------------------------
-
if settings is None:
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
- except KeyError: pass
+ except KeyError:
+ pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
- # --- backwards compatibility for scrapy.conf.settings singleton ---
- import warnings
- from scrapy.exceptions import ScrapyDeprecationWarning
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- from scrapy import conf
- conf.settings = settings
- # ------------------------------------------------------------------
-
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
- conflict_handler='resolve')
+ conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@@ -150,12 +146,14 @@ def execute(argv=None, settings=None):
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)
+
def _run_command(cmd, args, opts):
if opts.profile:
_run_command_profiled(cmd, args, opts)
else:
cmd.run(args, opts)
+
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
@@ -165,6 +163,7 @@ def _run_command_profiled(cmd, args, opts):
if opts.profile:
p.dump_stats(opts.profile)
+
if __name__ == '__main__':
try:
execute()
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 43b420821..0b24193c2 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -47,7 +47,7 @@ class ScrapyCommand(object):
def help(self):
"""An extensive help for the command. It will be shown when using the
- "help" command. It can contain newlines, since not post-formatting will
+ "help" command. It can contain newlines, since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index b8a9ef989..9d4437a47 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,12 +1,10 @@
-from __future__ import print_function
import time
-import sys
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
@@ -68,16 +66,17 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
- for spidername in args or spider_loader.list():
- spidercls = spider_loader.load(spidername)
- spidercls.start_requests = lambda s: conman.from_spider(s, result)
+ with set_environ(SCRAPY_CHECK='true'):
+ for spidername in args or spider_loader.list():
+ spidercls = spider_loader.load(spidername)
+ spidercls.start_requests = lambda s: conman.from_spider(s, result)
- tested_methods = conman.tested_methods_from_spidercls(spidercls)
- if opts.list:
- for method in tested_methods:
- contract_reqs[spidercls.name].append(method)
- elif tested_methods:
- self.crawler_process.crawl(spidercls)
+ tested_methods = conman.tested_methods_from_spidercls(spidercls)
+ if opts.list:
+ for method in tested_methods:
+ contract_reqs[spidercls.name].append(method)
+ elif tested_methods:
+ self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
@@ -95,4 +94,3 @@ class Command(ScrapyCommand):
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
-
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..724b4a1c4 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,5 @@
from __future__ import print_function
-import sys, six
+import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
@@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+
class Command(ScrapyCommand):
requires_project = False
@@ -45,8 +46,7 @@ class Command(ScrapyCommand):
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
- bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
- bytes_writer.write(bytes_ + b'\n')
+ sys.stdout.buffer.write(bytes_ + b'\n')
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..422183ac1 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,6 +1,7 @@
from __future__ import print_function
from scrapy.commands import ScrapyCommand
+
class Command(ScrapyCommand):
requires_project = True
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 69418a478..ef8acd29c 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,18 +51,21 @@ class Command(ScrapyCommand):
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
+ parser.add_option("--cbkwargs", dest="cbkwargs",
+ help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
-
@property
def max_level(self):
- levels = list(self.items.keys()) + list(self.requests.keys())
- if not levels:
- return 0
- return max(levels)
+ max_items, max_requests = 0, 0
+ if self.items:
+ max_items = max(self.items)
+ if self.requests:
+ max_requests = max(self.requests)
+ return max(max_items, max_requests)
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
@@ -83,9 +86,8 @@ class Command(ScrapyCommand):
def print_requests(self, lvl=None, colour=True):
if lvl is None:
- levels = list(self.requests.keys())
- if levels:
- requests = self.requests[max(levels)]
+ if self.requests:
+ requests = self.requests[max(self.requests)]
else:
requests = []
else:
@@ -111,10 +113,11 @@ class Command(ScrapyCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
- def run_callback(self, response, cb):
+ def run_callback(self, response, callback, cb_kwargs=None):
+ cb_kwargs = cb_kwargs or {}
items, requests = [], []
- for x in iterate_spider_output(cb(response)):
+ for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (BaseItem, dict)):
items.append(x)
elif isinstance(x, Request):
@@ -142,8 +145,7 @@ class Command(ScrapyCommand):
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
- logger.error('Unable to find spider for: %(url)s',
- {'url': url})
+ logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
@@ -160,7 +162,7 @@ class Command(ScrapyCommand):
{'url': url})
def prepare_request(self, spider, request, opts):
- def callback(response):
+ def callback(response, **cb_kwargs):
# memorize first request
if not self.first_response:
self.first_response = response
@@ -175,7 +177,7 @@ class Command(ScrapyCommand):
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
- {'url': response.url, 'spider': spider.name})
+ {'url': response.url, 'spider': spider.name})
return
else:
cb = 'parse'
@@ -192,7 +194,7 @@ class Command(ScrapyCommand):
# parse items and requests
depth = response.meta['_depth']
- items, requests = self.run_callback(response, cb)
+ items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
@@ -207,10 +209,14 @@ class Command(ScrapyCommand):
req.callback = callback
return requests
- #update request meta if any extra meta was passed through the --meta/-m opts.
+ # update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
request.meta.update(opts.meta)
+ # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+ if opts.cbkwargs:
+ request.cb_kwargs.update(opts.cbkwargs)
+
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.callback = callback
@@ -221,23 +227,29 @@ class Command(ScrapyCommand):
self.process_spider_arguments(opts)
self.process_request_meta(opts)
+ self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
-
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
-
if opts.meta:
try:
opts.meta = json.loads(opts.meta)
except ValueError:
- raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
- "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+ "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ def process_request_cb_kwargs(self, opts):
+ if opts.cbkwargs:
+ try:
+ opts.cbkwargs = json.loads(opts.cbkwargs)
+ except ValueError:
+ raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+ "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
def run(self, args, opts):
# parse arguments
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 376d3c84e..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -60,14 +60,13 @@ class Command(ScrapyCommand):
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
- valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
- if opts.output_format not in valid_output_formats:
+ if opts.output_format not in feed_exporters:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
- tuple(valid_output_formats)))
+ tuple(feed_exporters)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..ffe3aa2eb 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -4,6 +4,7 @@ import json
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 67337c26e..3b9f6eabb 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -119,4 +119,3 @@ class Command(ScrapyCommand):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')
-
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 577365c3b..8651948f7 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -30,4 +30,3 @@ class Command(ScrapyCommand):
print(patt % (name, version))
else:
print("Scrapy %s" % scrapy.__version__)
-
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 59e665016..41e77ba3b 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,6 +1,7 @@
-from scrapy.commands import fetch, ScrapyCommand
+from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
+
class Command(fetch.Command):
def short_desc(self):
diff --git a/scrapy/conf.py b/scrapy/conf.py
deleted file mode 100644
index 23efc6ffd..000000000
--- a/scrapy/conf.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# This module is kept for backwards compatibility, so users can import
-# scrapy.conf.settings and get the settings they expect
-
-import sys
-
-if 'scrapy.cmdline' not in sys.modules:
- from scrapy.utils.project import get_project_settings
- settings = get_project_settings()
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
- ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 259220a72..7b6591d86 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -90,11 +90,11 @@ class ContractsManager(object):
cb = request.callback
@wraps(cb)
- def cb_wrapper(response):
+ def cb_wrapper(response, **cb_kwargs):
try:
- output = cb(response)
+ output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
- except:
+ except Exception:
case = _create_testcase(method, 'callback')
results.addError(case, sys.exc_info())
@@ -121,7 +121,7 @@ class Contract(object):
cb = request.callback
@wraps(cb)
- def wrapper(response):
+ def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
@@ -133,7 +133,7 @@ class Contract(object):
else:
results.addSuccess(self.testcase_pre)
finally:
- return list(iterate_spider_output(cb(response)))
+ return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
@@ -144,8 +144,8 @@ class Contract(object):
cb = request.callback
@wraps(cb)
- def wrapper(response):
- output = list(iterate_spider_output(cb(response)))
+ def wrapper(response, **cb_kwargs):
+ output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 20582503d..e0d425874 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,8 +1,10 @@
+import json
+
from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
-from . import Contract
+from scrapy.contracts import Contract
# contracts
@@ -18,6 +20,20 @@ class UrlContract(Contract):
return args
+class CallbackKeywordArgumentsContract(Contract):
+ """ Contract to set the keyword arguments for the request.
+ The value should be a JSON-encoded dictionary, e.g.:
+
+ @cb_kwargs {"arg1": "some value"}
+ """
+
+ name = 'cb_kwargs'
+
+ def adjust_request_args(self, args):
+ args['cb_kwargs'] = json.loads(' '.join(self.args))
+ return args
+
+
class ReturnsContract(Contract):
""" Contract to check the output of a callback
@@ -84,6 +100,7 @@ class ScrapesContract(Contract):
def post_process(self, output):
for x in output:
if isinstance(x, (BaseItem, dict)):
- for arg in self.args:
- if not arg in x:
- raise ContractFail("'%s' field is missing" % arg)
+ missing = [arg for arg in self.args if arg not in x]
+ if missing:
+ raise ContractFail(
+ "Missing fields: %s" % ", ".join(missing))
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 59c3ad074..c5474a57f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,6 +1,4 @@
-from __future__ import absolute_import
import random
-import warnings
from time import time
from datetime import datetime
from collections import deque
@@ -12,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
@@ -60,10 +58,6 @@ class Slot(object):
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
- if hasattr(spider, 'DOWNLOAD_DELAY'):
- warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" %
- (type(spider).__name__, type(spider).__name__))
- delay = spider.DOWNLOAD_DELAY
if hasattr(spider, 'download_delay'):
delay = spider.download_delay
@@ -75,6 +69,8 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader(object):
+ DOWNLOAD_SLOT = 'download_slot'
+
def __init__(self, crawler):
self.settings = crawler.settings
self.signals = crawler.signals
@@ -111,8 +107,8 @@ class Downloader(object):
return key, self.slots[key]
def _get_slot_key(self, request, spider):
- if 'download_slot' in request.meta:
- return request.meta['download_slot']
+ if self.DOWNLOAD_SLOT in request.meta:
+ return request.meta[self.DOWNLOAD_SLOT]
key = urlparse_cached(request).hostname or ''
if self.ip_concurrency:
@@ -122,7 +118,7 @@ class Downloader(object):
def _enqueue_request(self, request, spider):
key, slot = self._get_slot(request, spider)
- request.meta['download_slot'] = key
+ request.meta[self.DOWNLOAD_SLOT] = key
def _deactivate(response):
slot.active.remove(request)
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 783d4c383..6e023ebcc 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,105 +1,93 @@
from OpenSSL import SSL
-from twisted.internet.ssl import ClientContextFactory
+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS
+from zope.interface.declarations import implementer
-from scrapy import twisted_version
-
-if twisted_version >= (14, 0, 0):
-
- from zope.interface.declarations import implementer
-
- from twisted.internet.ssl import (optionsForClientTLS,
- CertificateOptions,
- platformTrust)
- from twisted.web.client import BrowserLikePolicyForHTTPS
- from twisted.web.iweb import IPolicyForHTTPS
-
- from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
- @implementer(IPolicyForHTTPS)
- class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
- """
- Non-peer-certificate verifying HTTPS context factory
+@implementer(IPolicyForHTTPS)
+class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+ """
+ Non-peer-certificate verifying HTTPS context factory
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation
+ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+ which allows TLS protocol negotiation
- 'A TLS/SSL connection established with [this method] may
- understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
- """
+ 'A TLS/SSL connection established with [this method] may
+ understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+ """
- def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
- self._ssl_method = method
+ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
+ super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+ self._ssl_method = method
+ self.tls_verbose_logging = tls_verbose_logging
+ if tls_ciphers:
+ self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
+ else:
+ self.tls_ciphers = DEFAULT_CIPHERS
- def getCertificateOptions(self):
- # setting verify=True will require you to provide CAs
- # to verify against; in other words: it's not that simple
+ @classmethod
+ def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+ tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+ tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
+ return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
- # backward-compatible SSL/TLS method:
- #
- # * this will respect `method` attribute in often recommended
- # `ScrapyClientContextFactory` subclass
- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
- #
- # * getattr() for `_ssl_method` attribute for context factories
- # not calling super(..., self).__init__
- return CertificateOptions(verify=False,
- method=getattr(self, 'method',
- getattr(self, '_ssl_method', None)),
- fixBrokenPeers=True,
- acceptableCiphers=DEFAULT_CIPHERS)
+ def getCertificateOptions(self):
+ # setting verify=True will require you to provide CAs
+ # to verify against; in other words: it's not that simple
- # kept for old-style HTTP/1.0 downloader context twisted calls,
- # e.g. connectSSL()
- def getContext(self, hostname=None, port=None):
- return self.getCertificateOptions().getContext()
+ # backward-compatible SSL/TLS method:
+ #
+ # * this will respect `method` attribute in often recommended
+ # `ScrapyClientContextFactory` subclass
+ # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
+ #
+ # * getattr() for `_ssl_method` attribute for context factories
+ # not calling super(..., self).__init__
+ return CertificateOptions(verify=False,
+ method=getattr(self, 'method',
+ getattr(self, '_ssl_method', None)),
+ fixBrokenPeers=True,
+ acceptableCiphers=self.tls_ciphers)
- def creatorForNetloc(self, hostname, port):
- return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
+ # kept for old-style HTTP/1.0 downloader context twisted calls,
+ # e.g. connectSSL()
+ def getContext(self, hostname=None, port=None):
+ return self.getCertificateOptions().getContext()
+
+ def creatorForNetloc(self, hostname, port):
+ return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+ verbose_logging=self.tls_verbose_logging)
- @implementer(IPolicyForHTTPS)
- class BrowserLikeContextFactory(ScrapyClientContextFactory):
- """
- Twisted-recommended context factory for web clients.
+@implementer(IPolicyForHTTPS)
+class BrowserLikeContextFactory(ScrapyClientContextFactory):
+ """
+ Twisted-recommended context factory for web clients.
- Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
- "The default is to use a BrowserLikePolicyForHTTPS,
- so unless you have special requirements you can leave this as-is."
+ Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
- creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
- except this context factory allows setting the TLS/SSL method to use.
+ The default is to use a
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
+ have special requirements you can leave this as-is.
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation.
- """
- def creatorForNetloc(self, hostname, port):
+ :meth:`creatorForNetloc` is the same as
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
+ factory allows setting the TLS/SSL method to use.
- # trustRoot set to platformTrust() will use the platform's root CAs.
- #
- # This means that a website like https://www.cacert.org will be rejected
- # by default, since CAcert.org CA certificate is seldom shipped.
- return optionsForClientTLS(hostname.decode("ascii"),
- trustRoot=platformTrust(),
- extraCertificateOptions={
- 'method': self._ssl_method,
- })
+ The default OpenSSL method is ``TLS_METHOD`` (also called
+ ``SSLv23_METHOD``) which allows TLS protocol negotiation.
+ """
+ def creatorForNetloc(self, hostname, port):
-else:
-
- class ScrapyClientContextFactory(ClientContextFactory):
- "A SSL context factory which is more permissive against SSL bugs."
- # see https://github.com/scrapy/scrapy/issues/82
- # and https://github.com/scrapy/scrapy/issues/26
- # and https://github.com/scrapy/scrapy/issues/981
-
- def __init__(self, method=SSL.SSLv23_METHOD):
- self.method = method
-
- def getContext(self, hostname=None, port=None):
- ctx = ClientContextFactory.getContext(self)
- # Enable all workarounds to SSL bugs as documented by
- # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
- ctx.set_options(SSL.OP_ALL)
- return ctx
+ # trustRoot set to platformTrust() will use the platform's root CAs.
+ #
+ # This means that a website like https://www.cacert.org will be rejected
+ # by default, since CAcert.org CA certificate is seldom shipped.
+ return optionsForClientTLS(hostname.decode("ascii"),
+ trustRoot=platformTrust(),
+ extraCertificateOptions={
+ 'method': self._ssl_method,
+ })
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index ad25beb3b..9e5020753 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
- if (issubclass(respcls, TextResponse) and
- uri.media_type.split('/')[0] == 'text'):
+ if (issubclass(respcls, TextResponse)
+ and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index c342d4ab1..39ed67a1a 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol):
def close(self):
self.body.close() if self.filename else self.body.seek(0)
-_CODE_RE = re.compile("\d+")
+_CODE_RE = re.compile(r"\d+")
class FTPDownloadHandler(object):
@@ -112,4 +112,3 @@ class FTPDownloadHandler(object):
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=to_bytes(message))
raise result.type(result.value)
-
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index e4a7d8564..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,15 +1,4 @@
-from __future__ import absolute_import
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-
-
-# backwards compatibility
-class HttpDownloadHandler(HTTP10DownloadHandler):
-
- def __init__(self, *args, **kwargs):
- import warnings
- from scrapy.exceptions import ScrapyDeprecationWarning
- warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
- '.handlers.http10.HTTP10DownloadHandler instead',
- category=ScrapyDeprecationWarning, stacklevel=1)
- super(HttpDownloadHandler, self).__init__(*args, **kwargs)
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+ HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index d875fb1e4..be7298531 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,7 +1,7 @@
"""Download handlers for http and https schemes
"""
from twisted.internet import reactor
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.python import to_unicode
@@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object):
def __init__(self, settings):
self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ self._settings = settings
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
@@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object):
def _connect(self, factory):
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
- return reactor.connectSSL(host, port, factory,
- self.ClientContextFactory())
+ client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
+ return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 0673188a1..7d917cb74 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -13,21 +13,16 @@ from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from twisted.internet.error import TimeoutError
from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
- HTTPConnectionPool, ResponseFailed
-try:
- from twisted.web.client import URI
-except ImportError:
- from twisted.web.client import _URI as URI
+from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy import twisted_version
+
logger = logging.getLogger(__name__)
@@ -44,14 +39,23 @@ class HTTP11DownloadHandler(object):
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
- self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+ self._contextFactory = create_instance(
+ self._contextFactoryClass,
+ settings=settings,
+ crawler=None,
+ method=self._sslMethod,
+ )
except TypeError:
# use context factory defaults
- self._contextFactory = self._contextFactoryClass()
+ self._contextFactory = create_instance(
+ self._contextFactoryClass,
+ settings=settings,
+ crawler=None,
+ )
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD).\
- Please upgrade your context factory class to handle it or ignore it.""" % (
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
@@ -61,10 +65,13 @@ class HTTP11DownloadHandler(object):
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
- agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
+ agent = ScrapyAgent(
+ contextFactory=self._contextFactory,
+ pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
- fail_on_dataloss=self._fail_on_dataloss)
+ fail_on_dataloss=self._fail_on_dataloss,
+ )
return agent.download_request(request)
def close(self):
@@ -101,13 +108,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
- _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})')
+ _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})')
- def __init__(self, reactor, host, port, proxyConf, contextFactory,
- timeout=30, bindAddress=None):
+ def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
- super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost,
- proxyPort, timeout, bindAddress)
+ super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@@ -116,8 +121,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
- tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort,
- self._proxyAuthHeader)
+ tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@@ -140,16 +144,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocol.dataReceived = self._protocolDataReceived
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group('status')) == 200:
- try:
- # this sets proper Server Name Indication extension
- # but is only available for Twisted>=14.0
- sslOptions = self._contextFactory.creatorForNetloc(
- self._tunneledHost, self._tunneledPort)
- except AttributeError:
- # fall back to non-SNI SSL context factory
- sslOptions = self._contextFactory
- self._protocol.transport.startTLS(sslOptions,
- self._protocolFactory)
+ # set proper Server Name Indication extension
+ sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
+ self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
if respm:
@@ -167,8 +164,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
- connectDeferred = super(TunnelingTCP4ClientEndpoint,
- self).connect(protocolFactory)
+ connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@@ -178,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
r"""
Return binary content of a CONNECT request.
- >>> from scrapy.utils.python import to_native_str as s
+ >>> from scrapy.utils.python import to_unicode as s
>>> s(tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
@@ -205,42 +201,46 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
- super(TunnelingAgent, self).__init__(reactor, contextFactory,
- connectTimeout, bindAddress, pool)
+ super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
- if twisted_version >= (15, 0, 0):
- def _getEndpoint(self, uri):
- return TunnelingTCP4ClientEndpoint(
- self._reactor, uri.host, uri.port, self._proxyConf,
- self._contextFactory, self._endpointFactory._connectTimeout,
- self._endpointFactory._bindAddress)
- else:
- def _getEndpoint(self, scheme, host, port):
- return TunnelingTCP4ClientEndpoint(
- self._reactor, host, port, self._proxyConf,
- self._contextFactory, self._connectTimeout,
- self._bindAddress)
+ def _getEndpoint(self, uri):
+ return TunnelingTCP4ClientEndpoint(
+ reactor=self._reactor,
+ host=uri.host,
+ port=uri.port,
+ proxyConf=self._proxyConf,
+ contextFactory=self._contextFactory,
+ timeout=self._endpointFactory._connectTimeout,
+ bindAddress=self._endpointFactory._bindAddress,
+ )
- def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
- headers, bodyProducer, requestPath):
+ def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
- return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI,
- headers, bodyProducer, requestPath)
+ return super(TunnelingAgent, self)._requestWithEndpoint(
+ key=key,
+ endpoint=endpoint,
+ method=method,
+ parsedURI=parsedURI,
+ headers=headers,
+ bodyProducer=bodyProducer,
+ requestPath=requestPath,
+ )
class ScrapyProxyAgent(Agent):
- def __init__(self, reactor, proxyURI,
- connectTimeout=None, bindAddress=None, pool=None):
- super(ScrapyProxyAgent, self).__init__(reactor,
- connectTimeout=connectTimeout,
- bindAddress=bindAddress,
- pool=pool)
+ def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
+ super(ScrapyProxyAgent, self).__init__(
+ reactor=reactor,
+ connectTimeout=connectTimeout,
+ bindAddress=bindAddress,
+ pool=pool,
+ )
self._proxyURI = URI.fromBytes(proxyURI)
def request(self, method, uri, headers=None, bodyProducer=None):
@@ -249,16 +249,15 @@ class ScrapyProxyAgent(Agent):
"""
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
- if twisted_version >= (15, 0, 0):
- proxyEndpoint = self._getEndpoint(self._proxyURI)
- else:
- proxyEndpoint = self._getEndpoint(self._proxyURI.scheme,
- self._proxyURI.host,
- self._proxyURI.port)
- key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
- return self._requestWithEndpoint(key, proxyEndpoint, method,
- URI.fromBytes(uri), headers,
- bodyProducer, uri)
+ return self._requestWithEndpoint(
+ key=("http-proxy", self._proxyURI.host, self._proxyURI.port),
+ endpoint=self._getEndpoint(self._proxyURI),
+ method=method,
+ parsedURI=URI.fromBytes(uri),
+ headers=headers,
+ bodyProducer=bodyProducer,
+ requestPath=uri,
+ )
class ScrapyAgent(object):
@@ -286,18 +285,33 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
- if scheme == b'https' and not omitConnectTunnel:
- proxyConf = (proxyHost, proxyPort,
- request.headers.get(b'Proxy-Authorization', None))
- return self._TunnelingAgent(reactor, proxyConf,
- contextFactory=self._contextFactory, connectTimeout=timeout,
- bindAddress=bindaddress, pool=self._pool)
+ if scheme == b'https' and not omitConnectTunnel:
+ proxyAuth = request.headers.get(b'Proxy-Authorization', None)
+ proxyConf = (proxyHost, proxyPort, proxyAuth)
+ return self._TunnelingAgent(
+ reactor=reactor,
+ proxyConf=proxyConf,
+ contextFactory=self._contextFactory,
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
else:
- return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'),
- connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+ return self._ProxyAgent(
+ reactor=reactor,
+ proxyURI=to_bytes(proxy, encoding='ascii'),
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
- return self._Agent(reactor, contextFactory=self._contextFactory,
- connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+ return self._Agent(
+ reactor=reactor,
+ contextFactory=self._contextFactory,
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
def download_request(self, request):
timeout = request.meta.get('download_timeout') or self._connectTimeout
@@ -328,8 +342,7 @@ class ScrapyAgent(object):
else:
bodyproducer = None
start_time = time()
- d = agent.request(
- method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
+ d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@@ -384,8 +397,9 @@ class ScrapyAgent(object):
txresponse._transport._producer.abortConnection()
d = defer.Deferred(_cancel)
- txresponse.deliverBody(_ResponseReader(
- d, txresponse, request, maxsize, warnsize, fail_on_dataloss))
+ txresponse.deliverBody(
+ _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+ )
# save response for timeouts
self._txresponse = txresponse
@@ -420,22 +434,20 @@ class _RequestBodyProducer(object):
class _ResponseReader(protocol.Protocol):
- def __init__(self, finished, txresponse, request, maxsize, warnsize,
- fail_on_dataloss):
+ def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
self._finished = finished
self._txresponse = txresponse
self._request = request
self._bodybuf = BytesIO()
- self._maxsize = maxsize
- self._warnsize = warnsize
+ self._maxsize = maxsize
+ self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._fail_on_dataloss_warned = False
self._reached_warnsize = False
self._bytes_received = 0
def dataReceived(self, bodyBytes):
- # This maybe called several times after cancel was called with buffered
- # data.
+ # This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
@@ -448,8 +460,7 @@ class _ResponseReader(protocol.Protocol):
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
- # Clear buffer earlier to avoid keeping data in memory for a long
- # time.
+ # Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
@@ -479,10 +490,10 @@ class _ResponseReader(protocol.Protocol):
return
elif not self._fail_on_dataloss_warned:
- logger.warn("Got data loss in %s. If you want to process broken "
- "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
- " -- This message won't be shown in further requests",
- self._txresponse.request.absoluteURI.decode())
+ logger.warning("Got data loss in %s. If you want to process broken "
+ "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+ " -- This message won't be shown in further requests",
+ self._txresponse.request.absoluteURI.decode())
self._fail_on_dataloss_warned = True
self._finished.errback(reason)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..f4a42ce12 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
def _get_boto_connection():
@@ -21,7 +21,7 @@ def _get_boto_connection():
return http_request.headers
try:
- import boto.auth
+ import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index f5e2fca63..7a6a4dfac 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -7,6 +7,7 @@ import six
from twisted.internet import defer
+from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
@@ -35,12 +36,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def process_request(request):
for method in self.methods['process_request']:
response = yield method(request=request, spider=spider)
- assert response is None or isinstance(response, (Response, Request)), \
- 'Middleware %s.process_request must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
+ if response is not None and not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
+ (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
- defer.returnValue((yield download_func(request=request,spider=spider)))
+ defer.returnValue((yield download_func(request=request, spider=spider)))
@defer.inlineCallbacks
def process_response(response):
@@ -49,11 +50,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
defer.returnValue(response)
for method in self.methods['process_response']:
- response = yield method(request=request, response=response,
- spider=spider)
- assert isinstance(response, (Response, Request)), \
- 'Middleware %s.process_response must return Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response))
+ response = yield method(request=request, response=response, spider=spider)
+ if not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
+ (six.get_method_self(method).__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -62,11 +62,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def process_exception(_failure):
exception = _failure.value
for method in self.methods['process_exception']:
- response = yield method(request=request, exception=exception,
- spider=spider)
- assert response is None or isinstance(response, (Response, Request)), \
- 'Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response))
+ response = yield method(request=request, exception=exception, spider=spider)
+ if response is not None and not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
+ (six.get_method_self(method).__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index df8051182..4ed482058 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,17 +1,24 @@
import logging
+
from OpenSSL import SSL
+from service_identity.exceptions import CertificateError
+from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
+from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
+from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
+
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
METHOD_TLSv12 = 'TLSv1.2'
+
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
@@ -20,69 +27,66 @@ openssl_methods = {
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
}
-if twisted_version >= (14, 0, 0):
- # ClientTLSOptions requires a recent-enough version of Twisted.
- # Not having ScrapyClientTLSOptions should not matter for older
- # Twisted versions because it is not used in the fallback
- # ScrapyClientContextFactory.
- # taken from twisted/twisted/internet/_sslverify.py
-
- try:
- # XXX: this try-except is not needed in Twisted 17.0.0+ because
- # it requires pyOpenSSL 0.16+.
- from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
- except ImportError:
- SSL_CB_HANDSHAKE_START = 0x10
- SSL_CB_HANDSHAKE_DONE = 0x20
-
- from twisted.internet.ssl import AcceptableCiphers
- from twisted.internet._sslverify import (ClientTLSOptions,
- verifyHostname,
- VerificationError)
- try:
- # XXX: this import would fail on Debian jessie with system installed
- # service_identity library, due to lack of cryptography.x509 dependency
- # See https://github.com/pyca/service_identity/issues/21
- from service_identity.exceptions import CertificateError
- verification_errors = (CertificateError, VerificationError)
- except ImportError:
- verification_errors = VerificationError
-
- if twisted_version < (17, 0, 0):
- from twisted.internet._sslverify import _maybeSetHostNameIndication
- set_tlsext_host_name = _maybeSetHostNameIndication
- else:
- def set_tlsext_host_name(connection, hostNameBytes):
- connection.set_tlsext_host_name(hostNameBytes)
+if twisted_version < (17, 0, 0):
+ from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
+else:
+ def set_tlsext_host_name(connection, hostNameBytes):
+ connection.set_tlsext_host_name(hostNameBytes)
- class ScrapyClientTLSOptions(ClientTLSOptions):
- """
- SSL Client connection creator ignoring certificate verification errors
- (for genuinely invalid certificates or bugs in verification code).
+class ScrapyClientTLSOptions(ClientTLSOptions):
+ """
+ SSL Client connection creator ignoring certificate verification errors
+ (for genuinely invalid certificates or bugs in verification code).
- Same as Twisted's private _sslverify.ClientTLSOptions,
- except that VerificationError, CertificateError and ValueError
- exceptions are caught, so that the connection is not closed, only
- logging warnings.
- """
+ Same as Twisted's private _sslverify.ClientTLSOptions,
+ except that VerificationError, CertificateError and ValueError
+ exceptions are caught, so that the connection is not closed, only
+ logging warnings. Also, HTTPS connection parameters logging is added.
+ """
- def _identityVerifyingInfoCallback(self, connection, where, ret):
- if where & SSL_CB_HANDSHAKE_START:
- set_tlsext_host_name(connection, self._hostnameBytes)
- elif where & SSL_CB_HANDSHAKE_DONE:
- try:
- verifyHostname(connection, self._hostnameASCII)
- except verification_errors as e:
- logger.warning(
- 'Remote certificate is not valid for hostname "{}"; {}'.format(
- self._hostnameASCII, e))
+ def __init__(self, hostname, ctx, verbose_logging=False):
+ super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+ self.verbose_logging = verbose_logging
- except ValueError as e:
- logger.warning(
- 'Ignoring error while verifying certificate '
- 'from host "{}" (exception: {})'.format(
- self._hostnameASCII, repr(e)))
+ def _identityVerifyingInfoCallback(self, connection, where, ret):
+ if where & SSL.SSL_CB_HANDSHAKE_START:
+ set_tlsext_host_name(connection, self._hostnameBytes)
+ elif where & SSL.SSL_CB_HANDSHAKE_DONE:
+ if self.verbose_logging:
+ if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
+ if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
+ logger.debug('SSL connection to %s using protocol %s, cipher %s',
+ self._hostnameASCII,
+ connection.get_protocol_version_name(),
+ connection.get_cipher_name(),
+ )
+ else:
+ logger.debug('SSL connection to %s using cipher %s',
+ self._hostnameASCII,
+ connection.get_cipher_name(),
+ )
+ server_cert = connection.get_peer_certificate()
+ logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+ x509name_to_string(server_cert.get_issuer()),
+ x509name_to_string(server_cert.get_subject()),
+ )
+ key_info = get_temp_key_info(connection._ssl)
+ if key_info:
+ logger.debug('SSL temp key: %s', key_info)
- DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+ try:
+ verifyHostname(connection, self._hostnameASCII)
+ except (CertificateError, VerificationError) as e:
+ logger.warning(
+ 'Remote certificate is not valid for hostname "{}"; {}'.format(
+ self._hostnameASCII, e))
+
+ except ValueError as e:
+ logger.warning(
+ 'Ignoring error while verifying certificate '
+ 'from host "{}" (exception: {})'.format(
+ self._hostnameASCII, repr(e)))
+
+DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 1c89a0f9e..3fe13414a 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -95,7 +95,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
class ScrapyHTTPClientFactory(HTTPClientFactory):
"""Scrapy implementation of the HTTPClientFactory overwriting the
- serUrl method to make use of our Url object that cache the parse
+ setUrl method to make use of our Url object that cache the parse
result.
"""
@@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
-
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 37fe0a873..fa913e528 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -233,10 +233,11 @@ class ExecutionEngine(object):
def _on_success(response):
assert isinstance(response, (Response, Request))
if isinstance(response, Response):
- response.request = request # tie request to response received
+ response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
- self.signals.send_catch_log(signal=signals.response_received, \
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ self.signals.send_catch_log(signal=signals.response_received,
response=response, request=request, spider=spider)
return response
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index eb790a67e..975aede0c 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,19 +1,46 @@
import os
import json
import logging
+import warnings
from os.path import join, exists
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from queuelib import PriorityQueue
+
from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.job import job_dir
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+
logger = logging.getLogger(__name__)
class Scheduler(object):
+ """
+ Scrapy Scheduler. It allows to enqueue requests and then get
+ a next request to download. Scheduler is also handling duplication
+ filtering, via dupefilter.
+ Prioritization and queueing is not performed by the Scheduler.
+ User sets ``priority`` field for each Request, and a PriorityQueue
+ (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
+ to dequeue requests in a desired order.
+
+ Scheduler uses two PriorityQueue instances, configured to work in-memory
+ and on-disk (optional). When on-disk queue is present, it is used by
+ default, and an in-memory queue is used as a fallback for cases where
+ a disk queue can't handle a request (can't serialize it).
+
+ :setting:`SCHEDULER_MEMORY_QUEUE` and
+ :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
+ which PriorityQueue instances would be instantiated with, to keep requests
+ on disk and in memory respectively.
+
+ Overall, Scheduler is an object which holds several PriorityQueue instances
+ (in-memory and on-disk) and implements fallback logic for them.
+ Also, it handles dupefilters.
+ """
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
- logunser=False, stats=None, pqclass=None):
+ logunser=False, stats=None, pqclass=None, crawler=None):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.pqclass = pqclass
@@ -21,6 +48,7 @@ class Scheduler(object):
self.mqclass = mqclass
self.logunser = logunser
self.stats = stats
+ self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
@@ -28,26 +56,35 @@ class Scheduler(object):
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
dupefilter = create_instance(dupefilter_cls, settings, crawler)
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
+ if pqclass is PriorityQueue:
+ warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
+ " is no longer supported because of API changes; "
+ "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
+ ScrapyDeprecationWarning)
+ from scrapy.pqueues import ScrapyPriorityQueue
+ pqclass = ScrapyPriorityQueue
+
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
- logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
+ logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
+ settings.getbool('SCHEDULER_DEBUG'))
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
- stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
+ stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
+ mqclass=mqclass, crawler=crawler)
def has_pending_requests(self):
return len(self) > 0
def open(self, spider):
self.spider = spider
- self.mqs = self.pqclass(self._newmq)
+ self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason):
if self.dqs:
- prios = self.dqs.close()
- with open(join(self.dqdir, 'active.json'), 'w') as f:
- json.dump(prios, f)
+ state = self.dqs.close()
+ self._write_dqs_state(self.dqdir, state)
return self.df.close(reason)
def enqueue_request(self, request):
@@ -82,8 +119,7 @@ class Scheduler(object):
if self.dqs is None:
return
try:
- reqd = request_to_dict(request, self.spider)
- self.dqs.push(reqd, -request.priority)
+ self.dqs.push(request, -request.priority)
except ValueError as e: # non serializable request
if self.logunser:
msg = ("Unable to serialize request: %(request)s - reason:"
@@ -103,32 +139,51 @@ class Scheduler(object):
def _dqpop(self):
if self.dqs:
- d = self.dqs.pop()
- if d:
- return request_from_dict(d, self.spider)
+ return self.dqs.pop()
def _newmq(self, priority):
+ """ Factory for creating memory queues. """
return self.mqclass()
def _newdq(self, priority):
- return self.dqclass(join(self.dqdir, 'p%s' % priority))
+ """ Factory for creating disk queues. """
+ path = join(self.dqdir, 'p%s' % (priority, ))
+ return self.dqclass(path)
+
+ def _mq(self):
+ """ Create a new priority queue instance, with in-memory storage """
+ return create_instance(self.pqclass, None, self.crawler, self._newmq,
+ serialize=False)
def _dq(self):
- activef = join(self.dqdir, 'active.json')
- if exists(activef):
- with open(activef) as f:
- prios = json.load(f)
- else:
- prios = ()
- q = self.pqclass(self._newdq, startprios=prios)
+ """ Create a new priority queue instance, with disk storage """
+ state = self._read_dqs_state(self.dqdir)
+ q = create_instance(self.pqclass,
+ None,
+ self.crawler,
+ self._newdq,
+ state,
+ serialize=True)
if q:
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
{'queuesize': len(q)}, extra={'spider': self.spider})
return q
def _dqdir(self, jobdir):
+ """ Return a folder name to keep disk queue state at """
if jobdir:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
+
+ def _read_dqs_state(self, dqdir):
+ path = join(dqdir, 'active.json')
+ if not exists(path):
+ return ()
+ with open(path) as f:
+ return json.load(f)
+
+ def _write_dqs_state(self, dqdir, state):
+ with open(join(dqdir, 'active.json'), 'w') as f:
+ json.dump(state, f)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index ee1e95a0c..db463f989 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -135,7 +135,6 @@ class Scraper(object):
return self.spidermw.scrape_response(
self.call_spider, request_result, request, spider)
else:
- # FIXME: don't ignore errors in spider middleware
dfd = self.call_spider(request_result, request, spider)
return dfd.addErrback(
self._log_download_errors, request_result, request, spider)
@@ -143,7 +142,9 @@ class Scraper(object):
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
- dfd.addCallbacks(request.callback or spider.parse, request.errback)
+ dfd.addCallbacks(callback=request.callback or spider.parse,
+ errback=request.errback,
+ callbackKeywords=request.cb_kwargs)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
@@ -224,21 +225,22 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
- logger.error('Error processing %(item)s', {'item': item},
- exc_info=failure_to_exc_info(output),
- extra={'spider': spider})
+ logkws = self.logformatter.error(item, ex, response, spider)
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
+ exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
-
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 16b8435ab..00cee3ada 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,15 +3,21 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
+from itertools import chain, islice
+
import six
from twisted.python.failure import Failure
+from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.conf import build_component_list
+from scrapy.utils.python import MutableChain
+
def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__')
+
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@@ -24,15 +30,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
- if hasattr(mw, 'process_spider_output'):
- self.methods['process_spider_output'].appendleft(mw.process_spider_output)
- if hasattr(mw, 'process_spider_exception'):
- self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
+ self.methods['process_spider_output'].appendleft(getattr(mw, 'process_spider_output', None))
+ self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
def scrape_response(self, scrape_func, response, request, spider):
- fname = lambda f:'%s.%s' % (
+ fname = lambda f: '%s.%s' % (
six.get_method_self(f).__class__.__name__,
six.get_method_function(f).__name__)
@@ -40,36 +44,73 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
- assert result is None, \
- 'Middleware %s must returns None or ' \
- 'raise an exception, got %s ' \
- % (fname(method), type(result))
- except:
+ if result is not None:
+ raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
+ .format(fname(method), type(result)))
+ except _InvalidOutput:
+ raise
+ except Exception:
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
- def process_spider_exception(_failure):
+ def process_spider_exception(_failure, start_index=0):
exception = _failure.value
- for method in self.methods['process_spider_exception']:
+ # don't handle _InvalidOutput exception
+ if isinstance(exception, _InvalidOutput):
+ return _failure
+ method_list = islice(self.methods['process_spider_exception'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
result = method(response=response, exception=exception, spider=spider)
- assert result is None or _isiterable(result), \
- 'Middleware %s must returns None, or an iterable object, got %s ' % \
- (fname(method), type(result))
- if result is not None:
- return result
+ if _isiterable(result):
+ # stop exception handling by handing control over to the
+ # process_spider_output chain if an iterable has been returned
+ return process_spider_output(result, method_index+1)
+ elif result is None:
+ continue
+ else:
+ raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+ .format(fname(method), type(result)))
return _failure
- def process_spider_output(result):
- for method in self.methods['process_spider_output']:
- result = method(response=response, result=result, spider=spider)
- assert _isiterable(result), \
- 'Middleware %s must returns an iterable object, got %s ' % \
- (fname(method), type(result))
- return result
+ def process_spider_output(result, start_index=0):
+ # items in this iterable do not need to go through the process_spider_output
+ # chain, they went through it already from the process_spider_exception method
+ recovered = MutableChain()
+
+ def evaluate_iterable(iterable, index):
+ try:
+ for r in iterable:
+ yield r
+ except Exception as ex:
+ exception_result = process_spider_exception(Failure(ex), index+1)
+ if isinstance(exception_result, Failure):
+ raise
+ recovered.extend(exception_result)
+
+ method_list = islice(self.methods['process_spider_output'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
+ # the following might fail directly if the output value is not a generator
+ try:
+ result = method(response=response, result=result, spider=spider)
+ except Exception as ex:
+ exception_result = process_spider_exception(Failure(ex), method_index+1)
+ if isinstance(exception_result, Failure):
+ raise
+ return exception_result
+ if _isiterable(result):
+ result = evaluate_iterable(result, method_index)
+ else:
+ raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+ .format(fname(method), type(result)))
+
+ return chain(result, recovered)
dfd = mustbe_deferred(process_spider_input, response)
- dfd.addErrback(process_spider_exception)
- dfd.addCallback(process_spider_output)
+ dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception)
return dfd
def process_start_requests(self, start_requests, spider):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 04aee18ed..19b61dc7e 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,12 +1,13 @@
+import pprint
import six
import signal
import logging
import warnings
-import sys
from twisted.internet import reactor, defer
from zope.interface.verify import verifyClass, DoesNotImplement
+from scrapy import Spider
from scrapy.core.engine import ExecutionEngine
from scrapy.resolver import CachingThreadedResolver
from scrapy.interfaces import ISpiderLoader
@@ -27,6 +28,10 @@ logger = logging.getLogger(__name__)
class Crawler(object):
def __init__(self, spidercls, settings=None):
+ if isinstance(spidercls, Spider):
+ raise ValueError(
+ 'The spidercls argument must be a class, not an object')
+
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
@@ -34,14 +39,16 @@ class Crawler(object):
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
- d = dict(overridden_settings(self.settings))
- logger.info("Overridden settings: %(settings)r", {'settings': d})
-
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
+
+ d = dict(overridden_settings(self.settings))
+ logger.info("Overridden settings:\n%(settings)s",
+ {'settings': pprint.pformat(d)})
+
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
@@ -82,20 +89,9 @@ class Crawler(object):
yield self.engine.open_spider(self.spider, start_requests)
yield defer.maybeDeferred(self.engine.start)
except Exception:
- # In Python 2 reraising an exception after yield discards
- # the original traceback (see https://bugs.python.org/issue7563),
- # so sys.exc_info() workaround is used.
- # This workaround also works in Python 3, but it is not needed,
- # and it is slower, so in Python 3 we use native `raise`.
- if six.PY2:
- exc_info = sys.exc_info()
-
self.crawling = False
if self.engine is not None:
yield self.engine.close()
-
- if six.PY2:
- six.reraise(*exc_info)
raise
def _create_spider(self, *args, **kwargs):
@@ -106,6 +102,8 @@ class Crawler(object):
@defer.inlineCallbacks
def stop(self):
+ """Starts a graceful stop of the crawler and returns a deferred that is
+ fired when the crawler is stopped."""
if self.crawling:
self.crawling = False
yield defer.maybeDeferred(self.engine.stop)
@@ -114,7 +112,7 @@ class Crawler(object):
class CrawlerRunner(object):
"""
This is a convenient helper class that keeps track of, manages and runs
- crawlers inside an already setup Twisted `reactor`_.
+ crawlers inside an already setup :mod:`~twisted.internet.reactor`.
The CrawlerRunner object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -153,7 +151,7 @@ class CrawlerRunner(object):
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
keeping track of it so it can be stopped later.
- If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
+ If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
instance, this method will try to create one using this parameter as
the spider class given to it.
@@ -168,6 +166,10 @@ class CrawlerRunner(object):
:param dict kwargs: keyword arguments to initialize the spider
"""
+ if isinstance(crawler_or_spidercls, Spider):
+ raise ValueError(
+ 'The crawler_or_spidercls argument cannot be a spider object, '
+ 'it must be a spider class (or a Crawler object)')
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
@@ -188,13 +190,17 @@ class CrawlerRunner(object):
"""
Return a :class:`~scrapy.crawler.Crawler` object.
- * If `crawler_or_spidercls` is a Crawler, it is returned as-is.
- * If `crawler_or_spidercls` is a Spider subclass, a new Crawler
+ * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
+ * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
is constructed for it.
- * If `crawler_or_spidercls` is a string, this function finds
+ * If ``crawler_or_spidercls`` is a string, this function finds
a spider with this name in a Scrapy project (using spider loader),
then creates a Crawler instance for it.
"""
+ if isinstance(crawler_or_spidercls, Spider):
+ raise ValueError(
+ 'The crawler_or_spidercls argument cannot be a spider object, '
+ 'it must be a spider class (or a Crawler object)')
if isinstance(crawler_or_spidercls, Crawler):
return crawler_or_spidercls
return self._create_crawler(crawler_or_spidercls)
@@ -229,12 +235,13 @@ class CrawlerProcess(CrawlerRunner):
A class to run multiple scrapy crawlers in a process simultaneously.
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
- for starting a Twisted `reactor`_ and handling shutdown signals, like the
- keyboard interrupt command Ctrl-C. It also configures top-level logging.
+ for starting a :mod:`~twisted.internet.reactor` and handling shutdown
+ signals, like the keyboard interrupt command Ctrl-C. It also configures
+ top-level logging.
This utility should be a better fit than
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
- Twisted `reactor`_ within your application.
+ :mod:`~twisted.internet.reactor` within your application.
The CrawlerProcess object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -269,11 +276,11 @@ class CrawlerProcess(CrawlerRunner):
def start(self, stop_after_crawl=True):
"""
- This method starts a Twisted `reactor`_, adjusts its pool size to
- :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
- on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
+ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
+ size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
+ based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
- If `stop_after_crawl` is True, the reactor will be stopped after all
+ If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
@@ -317,14 +324,7 @@ class CrawlerProcess(CrawlerRunner):
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
- if settings.get('SPIDER_MANAGER_CLASS'):
- warnings.warn(
- 'SPIDER_MANAGER_CLASS option is deprecated. '
- 'Please use SPIDER_LOADER_CLASS.',
- category=ScrapyDeprecationWarning, stacklevel=2
- )
- cls_path = settings.get('SPIDER_MANAGER_CLASS',
- settings.get('SPIDER_LOADER_CLASS'))
+ cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
try:
verifyClass(ISpiderLoader, loader_cls)
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..ba50793bb 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(six.u(r''))
+
+
def _has_ajaxcrawlable_meta(text):
"""
>>> _has_ajaxcrawlable_meta('')
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..aeb7578b8 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,12 +1,12 @@
-import os
-import six
import logging
from collections import defaultdict
+import six
+
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
def _debug_cookie(self, request, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
def _debug_set_cookie(self, response, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 49313cc04..e2d73f347 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
+from io import BytesIO
import zipfile
import tarfile
import logging
@@ -11,11 +12,6 @@ from tempfile import mktemp
import six
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
-
from scrapy.responsetypes import responsetypes
logger = logging.getLogger(__name__)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 495b103d1..4e06f8236 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,11 +1,19 @@
from email.utils import formatdate
+
from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
- ConnectionRefusedError, ConnectionDone, ConnectError, \
- ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+ ConnectError,
+ ConnectionDone,
+ ConnectionLost,
+ ConnectionRefusedError,
+ DNSLookupError,
+ TCPTimedOutError,
+ TimeoutError,
+)
from twisted.web.client import ResponseFailed
+
from scrapy import signals
-from scrapy.exceptions import NotConfigured, IgnoreRequest
+from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.misc import load_object
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2c35d1b90..5e4542b6c 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,10 +1,8 @@
import base64
+from urllib.request import _parse_proxy
+
from six.moves.urllib.parse import unquote, urlunparse
from six.moves.urllib.request import getproxies, proxy_bypass
-try:
- from urllib2 import _parse_proxy
-except ImportError:
- from urllib.request import _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 30cae3fee..b73f864dd 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
import logging
-from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@@ -34,6 +34,8 @@ class BaseRedirectMiddleware(object):
redirected.meta['redirect_ttl'] = ttl - 1
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
[request.url]
+ redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
+ [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -68,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if 'Location' not in response.headers or response.status not in allowed_status:
return response
- location = safe_url_string(response.headers['location'])
+ location = safe_url_string(response.headers['Location'])
+ if response.headers['Location'].startswith(b'//'):
+ request_scheme = urlparse(request.url).scheme
+ location = request_scheme + '://' + location.lstrip('/')
redirected_url = urljoin(request.url, location)
@@ -86,6 +91,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
+ self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
settings.getint('METAREFRESH_MAXDELAY'))
@@ -94,7 +100,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
not isinstance(response, HtmlResponse):
return response
- interval, url = get_meta_refresh(response)
+ interval, url = get_meta_refresh(response,
+ ignore_tags=self._ignore_tags)
if url and interval < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 07e979628..dbc605a4c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non failed) pages. Once
-there is no more failed pages to retry this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
+once the spider has finished crawling all regular (non failed) pages.
"""
import logging
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 200245210..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -6,14 +6,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
import logging
-from six.moves.urllib import robotparser
-
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
+from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
@@ -24,10 +22,14 @@ class RobotsTxtMiddleware(object):
def __init__(self, crawler):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
raise NotConfigured
-
+ self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
+ self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
self.crawler = crawler
- self._useragent = crawler.settings.get('USER_AGENT')
self._parsers = {}
+ self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+
+ # check if parser dependencies are met, this should throw an error otherwise.
+ self._parserimpl.from_crawler(self.crawler, b'')
@classmethod
def from_crawler(cls, crawler):
@@ -43,7 +45,11 @@ class RobotsTxtMiddleware(object):
def process_request_2(self, rp, request, spider):
if rp is None:
return
- if not rp.can_fetch(to_native_str(self._useragent), request.url):
+
+ useragent = self._robotstxt_useragent
+ if not useragent:
+ useragent = request.headers.get(b'User-Agent', self._default_useragent)
+ if not rp.allowed(request.url, useragent):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
self.crawler.stats.inc_value('robotstxt/forbidden')
@@ -62,13 +68,14 @@ class RobotsTxtMiddleware(object):
meta={'dont_obey_robotstxt': True}
)
dfd = self.crawler.engine.download(robotsreq, spider)
- dfd.addCallback(self._parse_robots, netloc)
+ dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value('robotstxt/request_count')
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
+
def cb(result):
d.callback(result)
return result
@@ -85,27 +92,10 @@ class RobotsTxtMiddleware(object):
extra={'spider': spider})
return failure
- def _parse_robots(self, response, netloc):
+ def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
- self.crawler.stats.inc_value(
- 'robotstxt/response_status_count/{}'.format(response.status))
- rp = robotparser.RobotFileParser(response.url)
- body = ''
- if hasattr(response, 'text'):
- body = response.text
- else: # last effort try
- try:
- body = response.body.decode('utf-8')
- except UnicodeDecodeError:
- # If we found garbage, disregard it:,
- # but keep the lookup cached (in self._parsers)
- # Running rp.parse() will set rp state from
- # 'disallow all' to 'allow any'.
- self.crawler.stats.inc_value('robotstxt/unicode_error_count')
- # stdlib's robotparser expects native 'str' ;
- # with unicode input, non-ASCII encoded bytes decoding fails in Python2
- rp.parse(to_native_str(body).splitlines())
-
+ self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+ rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
rp_dfd.callback(rp)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 0bcdd3495..4d95eb847 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -5,6 +5,7 @@ import logging
from scrapy.utils.job import job_dir
from scrapy.utils.request import referer_str, request_fingerprint
+
class BaseDupeFilter(object):
@classmethod
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 4bcecd994..7c4bb3d00 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -7,19 +7,32 @@ new exceptions here without documenting them there.
# Internal
+
class NotConfigured(Exception):
"""Indicates a missing configuration situation"""
pass
+
+class _InvalidOutput(TypeError):
+ """
+ Indicates an invalid value has been returned by a middleware's processing method.
+ Internal and undocumented, it should not be raised or caught by user code.
+ """
+ pass
+
+
# HTTP and crawling
+
class IgnoreRequest(Exception):
"""Indicates a decision was made not to process a request"""
+
class DontCloseSpider(Exception):
"""Request the spider not to be closed yet"""
pass
+
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
@@ -27,30 +40,37 @@ class CloseSpider(Exception):
super(CloseSpider, self).__init__()
self.reason = reason
+
# Items
+
class DropItem(Exception):
"""Drop item from the item pipeline"""
pass
+
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
pass
+
# Commands
+
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
+
class ScrapyDeprecationWarning(Warning):
"""Warning category for deprecated features, since the default
DeprecationWarning is silenced on Python 2.7+
"""
pass
+
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
pass
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 695c74fec..3defafd60 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
-import sys
import pprint
import marshal
import six
@@ -12,7 +11,7 @@ from six.moves import cPickle as pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
@@ -31,7 +30,7 @@ class BaseItemExporter(object):
def _configure(self, options, dont_fail=False):
"""Configure the exporter by poping options from the ``options`` dict.
If dont_fail is set, it won't raise an exception on unexpected options
- (useful for using with keyword arguments in subclasses constructors)
+ (useful for using with keyword arguments in subclasses ``__init__`` methods)
"""
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
@@ -143,11 +142,11 @@ class XmlItemExporter(BaseItemExporter):
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
- self._xg_characters('\n')
+ self.xg.characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
- self._xg_characters(' ' * self.indent * depth)
+ self.xg.characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
@@ -182,26 +181,12 @@ class XmlItemExporter(BaseItemExporter):
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
- self._xg_characters(serialized_value)
+ self.xg.characters(serialized_value)
else:
- self._xg_characters(str(serialized_value))
+ self.xg.characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
- # Workaround for https://bugs.python.org/issue17606
- # Before Python 2.7.4 xml.sax.saxutils required bytes;
- # since 2.7.4 it requires unicode. The bug is likely to be
- # fixed in 2.7.6, but 2.7.6 will still support unicode,
- # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
- if sys.version_info[:3] >= (2, 7, 4):
- def _xg_characters(self, serialized_value):
- if not isinstance(serialized_value, six.text_type):
- serialized_value = serialized_value.decode(self.encoding)
- return self.xg.characters(serialized_value)
- else: # pragma: no cover
- def _xg_characters(self, serialized_value):
- return self.xg.characters(serialized_value)
-
class CsvItemExporter(BaseItemExporter):
@@ -216,7 +201,7 @@ class CsvItemExporter(BaseItemExporter):
write_through=True,
encoding=self.encoding,
newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
- ) if six.PY3 else file
+ )
self.csv_writer = csv.writer(self.stream, **kwargs)
self._headers_not_written = True
self._join_multivalued = join_multivalued
@@ -246,7 +231,7 @@ class CsvItemExporter(BaseItemExporter):
def _build_row(self, values):
for s in values:
try:
- yield to_native_str(s, self.encoding)
+ yield to_unicode(s, self.encoding)
except TypeError:
yield s
@@ -276,6 +261,13 @@ class PickleItemExporter(BaseItemExporter):
class MarshalItemExporter(BaseItemExporter):
+ """Exports items in a Python-specific binary format (see
+ :mod:`marshal`).
+
+ :param file: The file-like object to use for exporting the data. Its
+ ``write`` method should accept :class:`bytes` (a disk file
+ opened in binary mode, a :class:`~io.BytesIO` object, etc)
+ """
def __init__(self, file, **kwargs):
self._configure(kwargs)
@@ -297,10 +289,13 @@ class PprintItemExporter(BaseItemExporter):
class PythonItemExporter(BaseItemExporter):
- """The idea behind this exporter is to have a mechanism to serialize items
- to built-in python types so any serialization library (like
- json, msgpack, binc, etc) can be used on top of it. Its main goal is to
- seamless support what BaseItemExporter does plus nested items.
+ """This is a base class for item exporters that extends
+ :class:`BaseItemExporter` with support for nested items.
+
+ It serializes items to built-in Python types, so that any serialization
+ library (e.g. :mod:`json` or msgpack_) can be used on top of it.
+
+ .. _msgpack: https://pypi.org/project/msgpack/
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
diff --git a/scrapy/extension.py b/scrapy/extension.py
index e39e456fa..050b87e5f 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
+
class ExtensionManager(MiddlewareManager):
component_name = 'extension'
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 3d9a307b7..20adfbe4b 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,14 +1,16 @@
"""
Extension for collecting core stats like items scraped and start/finish times
"""
-import datetime
+from datetime import datetime
from scrapy import signals
+
class CoreStats(object):
def __init__(self, stats):
self.stats = stats
+ self.start_time = None
@classmethod
def from_crawler(cls, crawler):
@@ -21,10 +23,15 @@ class CoreStats(object):
return o
def spider_opened(self, spider):
- self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+ self.start_time = datetime.utcnow()
+ self.stats.set_value('start_time', self.start_time, spider=spider)
def spider_closed(self, spider, reason):
- self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+ finish_time = datetime.utcnow()
+ elapsed_time = finish_time - self.start_time
+ elapsed_time_seconds = elapsed_time.total_seconds()
+ self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
+ self.stats.set_value('finish_time', finish_time, spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 22ebf3b3f..e2492d506 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,8 +10,7 @@ import logging
import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
-import six
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, unquote
from ftplib import FTP
from zope.interface import Interface, implementer
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
def __init__(self, uri, _stdout=None):
if not _stdout:
- _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+ _stdout = sys.stdout.buffer
self._stdout = _stdout
def open(self, spider):
@@ -93,12 +92,13 @@ class FileFeedStorage(object):
class S3FeedStorage(BlockingFeedStorage):
- def __init__(self, uri, access_key=None, secret_key=None):
- # BEGIN Backwards compatibility for initialising without keys (and
+ def __init__(self, uri, access_key=None, secret_key=None, acl=None):
+ # BEGIN Backward compatibility for initialising without keys (and
# without using from_crawler)
no_defaults = access_key is None and secret_key is None
if no_defaults:
- from scrapy.conf import settings
+ from scrapy.utils.project import get_project_settings
+ settings = get_project_settings()
if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -111,13 +111,14 @@ class S3FeedStorage(BlockingFeedStorage):
)
access_key = settings['AWS_ACCESS_KEY_ID']
secret_key = settings['AWS_SECRET_ACCESS_KEY']
- # END Backwards compatibility
+ # END Backward compatibility
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
+ self.acl = acl
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
@@ -130,37 +131,54 @@ class S3FeedStorage(BlockingFeedStorage):
@classmethod
def from_crawler(cls, crawler, uri):
- return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],
- crawler.settings['AWS_SECRET_ACCESS_KEY'])
+ return cls(
+ uri=uri,
+ access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
+ secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
+ acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
+ )
def _store_in_thread(self, file):
file.seek(0)
if self.is_botocore:
+ kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
- Bucket=self.bucketname, Key=self.keyname, Body=file)
+ Bucket=self.bucketname, Key=self.keyname, Body=file,
+ **kwargs)
else:
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
- key.set_contents_from_file(file)
+ kwargs = {'policy': self.acl} if self.acl else {}
+ key.set_contents_from_file(file, **kwargs)
key.close()
class FTPFeedStorage(BlockingFeedStorage):
- def __init__(self, uri):
+ def __init__(self, uri, use_active_mode=False):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.username = u.username
- self.password = u.password
+ self.password = unquote(u.password)
self.path = u.path
+ self.use_active_mode = use_active_mode
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return cls(
+ uri=uri,
+ use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+ )
def _store_in_thread(self, file):
file.seek(0)
ftp = FTP()
ftp.connect(self.host, self.port)
ftp.login(self.username, self.password)
+ if self.use_active_mode:
+ ftp.set_pasv(False)
dirname, filename = posixpath.split(self.path)
ftp_makedirs_cwd(ftp, dirname)
ftp.storbinary('STOR %s' % filename, file)
@@ -180,9 +198,9 @@ class FeedExporter(object):
def __init__(self, settings):
self.settings = settings
- self.urifmt = settings['FEED_URI']
- if not self.urifmt:
+ if not settings['FEED_URI']:
raise NotConfigured
+ self.urifmt = str(settings['FEED_URI'])
self.format = settings['FEED_FORMAT'].lower()
self.export_encoding = settings['FEED_EXPORT_ENCODING']
self.storages = self._load_components('FEED_STORAGES')
@@ -223,7 +241,9 @@ class FeedExporter(object):
def close_spider(self, spider):
slot = self.slot
if not slot.itemcount and not self.store_empty:
- return
+ # We need to call slot.storage.store nonetheless to get the file
+ # properly closed.
+ return defer.maybeDeferred(slot.storage.store, slot.file)
if self._exporting:
slot.exporter.finish_exporting()
self._exporting = False
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 1b5e05b1b..11403957c 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,19 +1,22 @@
from __future__ import print_function
-import os
+
import gzip
import logging
-from six.moves import cPickle as pickle
+import os
+from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
-from email.utils import mktime_tz, parsedate_tz
+
+from six.moves import cPickle as pickle
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
+
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
-from scrapy.utils.request import request_fingerprint
-from scrapy.utils.project import data_path
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
+from scrapy.utils.project import data_path
+from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@@ -31,7 +34,7 @@ class DummyPolicy(object):
def should_cache_response(self, response, request):
return response.status not in self.ignore_http_codes
- def is_cached_response_fresh(self, response, request):
+ def is_cached_response_fresh(self, cachedresponse, request):
return True
def is_cached_response_valid(self, cachedresponse, response, request):
@@ -70,7 +73,7 @@ class RFC2616Policy(object):
return True
def should_cache_response(self, response, request):
- # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
+ # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
# Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
# Status code 206 is not included because cache can not deal with partial contents
cc = self._parse_cachecontrol(response)
@@ -342,75 +345,6 @@ class FilesystemCacheStorage(object):
return pickle.load(f)
-class LeveldbCacheStorage(object):
-
- def __init__(self, settings):
- import leveldb
- self._leveldb = leveldb
- self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
- self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
- self.db = None
-
- def open_spider(self, spider):
- dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
- self.db = self._leveldb.LevelDB(dbpath)
-
- logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
-
- def close_spider(self, spider):
- # Do compactation each time to save space and also recreate files to
- # avoid them being removed in storages with timestamp-based autoremoval.
- self.db.CompactRange()
- del self.db
- garbage_collect()
-
- def retrieve_response(self, spider, request):
- data = self._read_data(spider, request)
- if data is None:
- return # not cached
- url = data['url']
- status = data['status']
- headers = Headers(data['headers'])
- body = data['body']
- respcls = responsetypes.from_args(headers=headers, url=url)
- response = respcls(url=url, headers=headers, status=status, body=body)
- return response
-
- def store_response(self, spider, request, response):
- key = self._request_key(request)
- data = {
- 'status': response.status,
- 'url': response.url,
- 'headers': dict(response.headers),
- 'body': response.body,
- }
- batch = self._leveldb.WriteBatch()
- batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
- batch.Put(key + b'_time', to_bytes(str(time())))
- self.db.Write(batch)
-
- def _read_data(self, spider, request):
- key = self._request_key(request)
- try:
- ts = self.db.Get(key + b'_time')
- except KeyError:
- return # not found or invalid entry
-
- if 0 < self.expiration_secs < time() - float(ts):
- return # expired
-
- try:
- data = self.db.Get(key + b'_data')
- except KeyError:
- return # invalid entry
- else:
- return pickle.loads(data)
-
- def _request_key(self, request):
- return to_bytes(request_fingerprint(request))
-
-
-
def parse_cachecontrol(header):
"""Parse Cache-Control header
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2220cbd8f..8ba770ec0 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -5,6 +5,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
+
class SpiderState(object):
"""Store and load spider state during a scraping job"""
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index f04a9d3e5..e6c58e1f1 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -10,6 +10,7 @@ from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.http.request.form import FormRequest
from scrapy.http.request.rpc import XmlRpcRequest
+from scrapy.http.request.json_request import JsonRequest
from scrapy.http.response import Response
from scrapy.http.response.html import HtmlResponse
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4e8056750..60a14c6f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
)
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
return name in self.request.headers
def get_header(self, name, default=None):
- return to_native_str(self.request.headers.get(name, default),
- errors='replace')
+ return to_unicode(self.request.headers.get(name, default),
+ errors='replace')
def header_items(self):
return [
- (to_native_str(k, errors='replace'),
- [to_native_str(x, errors='replace') for x in v])
+ (to_unicode(k, errors='replace'),
+ [to_unicode(x, errors='replace') for x in v])
for k, v in self.request.headers.items()
]
@@ -189,7 +189,7 @@ class WrappedResponse(object):
# python3 cookiejars calls get_all
def get_all(self, name, default=None):
- return [to_native_str(v, errors='replace')
+ return [to_unicode(v, errors='replace')
for v in self.response.headers.getlist(name)]
# python2 cookiejars calls getheaders
getheaders = get_all
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..f3b46b994 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -91,5 +91,3 @@ class Headers(CaselessDict):
def __copy__(self):
return self.__class__(self)
copy = __copy__
-
-
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index cd4360483..76a428199 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -12,13 +12,14 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
from scrapy.http.common import obsolete_setter
+from scrapy.utils.curl import curl_to_request_kwargs
class Request(object_ref):
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, flags=None):
+ dont_filter=False, errback=None, flags=None, cb_kwargs=None):
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
@@ -40,8 +41,15 @@ class Request(object_ref):
self.dont_filter = dont_filter
self._meta = dict(meta) if meta else None
+ self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
self.flags = [] if flags is None else list(flags)
+ @property
+ def cb_kwargs(self):
+ if self._cb_kwargs is None:
+ self._cb_kwargs = {}
+ return self._cb_kwargs
+
@property
def meta(self):
if self._meta is None:
@@ -58,7 +66,7 @@ class Request(object_ref):
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
- if ':' not in self._url:
+ if ('://' not in self._url) and (not self._url.startswith('data:')):
raise ValueError('Missing scheme in request url: %s' % self._url)
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@@ -92,7 +100,38 @@ class Request(object_ref):
given new values.
"""
for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
- 'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
+ 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
+
+ @classmethod
+ def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+ """Create a Request object from a string containing a `cURL
+ `_ command. It populates the HTTP method, the
+ URL, the headers, the cookies and the body. It accepts the same
+ arguments as the :class:`Request` class, taking preference and
+ overriding the values of the same arguments contained in the cURL
+ command.
+
+ Unrecognized options are ignored by default. To raise an error when
+ finding unknown options call this method by passing
+ ``ignore_unknown_options=False``.
+
+ .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
+ subclasses, such as :class:`~scrapy.http.JSONRequest`, or
+ :class:`~scrapy.http.XmlRpcRequest`, as well as having
+ :ref:`downloader middlewares `
+ and
+ :ref:`spider middlewares `
+ enabled, such as
+ :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`,
+ :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`,
+ or
+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
+ may modify the :class:`~scrapy.http.Request` object.
+
+ """
+ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
+ request_kwargs.update(kwargs)
+ return cls(**request_kwargs)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index c2413b431..b6feede07 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -18,6 +18,7 @@ from scrapy.utils.response import get_base_url
class FormRequest(Request):
+ valid_form_methods = ['GET', 'POST']
def __init__(self, *args, **kwargs):
formdata = kwargs.pop('formdata', None)
@@ -48,7 +49,13 @@ class FormRequest(Request):
form = _get_form(response, formname, formid, formnumber, formxpath)
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
url = _get_form_url(form, kwargs.pop('url', None))
+
method = kwargs.pop('method', form.method)
+ if method is not None:
+ method = method.upper()
+ if method not in cls.valid_form_methods:
+ method = 'GET'
+
return cls(url=url, method=method, formdata=formdata, **kwargs)
@@ -97,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
el = el.getparent()
if el is None:
break
- encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
- raise ValueError('No