diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 36484c49f..6e7be142e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,8 @@ [bumpversion] -current_version = 1.3.2 +current_version = 1.5.0 commit = True tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] + diff --git a/.travis.yml b/.travis.yml index 2df02ea43..6635f5d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,32 +11,32 @@ matrix: env: TOXENV=py27 - python: 2.7 env: TOXENV=jessie - - python: 3.3 - env: TOXENV=py33 + - python: 2.7 + env: TOXENV=pypy + - python: 2.7 + env: TOXENV=pypy3 + - python: 3.4 + env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 2.7 - env: TOXENV=pypy - python: 3.6 env: TOXENV=docs - allow_failures: - - python: 2.7 - env: TOXENV=pypy install: - | if [ "$TOXENV" = "pypy" ]; then - export PYENV_ROOT="$HOME/.pyenv" - if [ -f "$PYENV_ROOT/bin/pyenv" ]; then - pushd "$PYENV_ROOT" && git pull && popd - else - rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" - fi - # get latest PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1` - "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" - virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + if [ "$TOXENV" = "pypy3" ]; then + export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - pip install -U tox twine wheel codecov diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 162602248..d477168eb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -3,7 +3,7 @@ ## Our Pledge In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and +contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88c472f6f..0a11b05d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ The guidelines for contributing are available here: -http://doc.scrapy.org/en/master/contributing.html +https://doc.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the -support channels to get it answered: http://scrapy.org/community/ +support channels to get it answered: https://scrapy.org/community/ diff --git a/INSTALL b/INSTALL index 84803a933..a3c7899c6 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* http://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://doc.scrapy.org/en/latest/intro/install.html (online version) diff --git a/LICENSE b/LICENSE index 68ccf9762..4d0a0863a 100644 --- a/LICENSE +++ b/LICENSE @@ -4,11 +4,11 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of Scrapy nor the names of its contributors may be used diff --git a/README.rst b/README.rst index 4eb36b44a..1361eac26 100644 --- a/README.rst +++ b/README.rst @@ -6,8 +6,12 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: PyPI Version +.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg + :target: https://pypi.python.org/pypi/Scrapy + :alt: Supported Python Versions + .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: http://travis-ci.org/scrapy/scrapy + :target: https://travis-ci.org/scrapy/scrapy :alt: Build Status .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg @@ -15,7 +19,7 @@ Scrapy :alt: Wheel Status .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg - :target: http://codecov.io/github/scrapy/scrapy?branch=master + :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report .. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg @@ -31,12 +35,12 @@ crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. For more information including a list of features check the Scrapy homepage at: -http://scrapy.org +https://scrapy.org Requirements ============ -* Python 2.7 or Python 3.3+ +* Python 2.7 or Python 3.4+ * Works on Linux, Windows, Mac OSX, BSD Install @@ -47,29 +51,28 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -http://doc.scrapy.org/en/latest/intro/install.html - -Releases -======== - -You can download the latest stable and development releases from: -http://scrapy.org/download/ +https://doc.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at http://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` directory. +Releases +======== + +You can find release notes at https://doc.scrapy.org/en/latest/news.html + Community (blog, twitter, mail list, IRC) ========================================= -See http://scrapy.org/community/ +See https://scrapy.org/community/ Contributing ============ -See http://doc.scrapy.org/en/master/contributing.html +See https://doc.scrapy.org/en/master/contributing.html Code of Conduct --------------- @@ -83,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com. Companies using Scrapy ====================== -See http://scrapy.org/companies/ +See https://scrapy.org/companies/ Commercial Support ================== -See http://scrapy.org/support/ +See https://scrapy.org/support/ diff --git a/artwork/README.rst b/artwork/README.rst index 016462f2c..92f6ecb7e 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -10,10 +10,10 @@ scrapy-logo.jpg Main Scrapy logo, in JPEG format. -qlassik.zip +qlassik.zip ----------- -Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font +Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font scrapy-blog.logo.xcf -------------------- diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..d8aa6b984 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,6 @@ +comment: + layout: "header, diff, tree" + +coverage: + status: + project: false diff --git a/debian/control b/debian/control index f3a31753b..2cc8eedf4 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: Scrapinghub Team Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) Standards-Version: 3.8.4 -Homepage: http://scrapy.org/ +Homepage: https://scrapy.org/ Package: scrapy Architecture: all @@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25 Provides: python-scrapy, scrapy-0.25 Description: Python web crawling and web scraping framework Scrapy is a fast high-level web crawling and web scraping framework, - used to crawl websites and extract structured data from their pages. - It can be used for a wide range of purposes, from data mining to + used to crawl websites and extract structured data from their pages. + It can be used for a wide range of purposes, from data mining to monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright index 4cc239002..c1bf47565 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,6 +1,6 @@ This package was debianized by the Scrapinghub team . -It was downloaded from http://scrapy.org +It was downloaded from https://scrapy.org Upstream Author: Scrapy Developers @@ -14,10 +14,10 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. diff --git a/docs/Makefile b/docs/Makefile index eaba3ba2b..187f03c4c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,7 +10,8 @@ PAPER = SOURCES = SHELL = /bin/bash -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ + -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help update build html htmlhelp clean @@ -82,7 +83,8 @@ pydoc-topics: build "into the Lib/ directory" htmlview: html - $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" + $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ + os.path.realpath('build/html/index.html'))" clean: -rm -rf build/* diff --git a/docs/README.rst b/docs/README.rst index 733af2af4..0a343cd19 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -11,11 +11,11 @@ Setup the environment --------------------- To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run +and all its dependencies run the following command from this dir :: - pip install 'Sphinx >= 1.3' + pip install -r requirements.txt Compile the documentation diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 83b0d2cc6..192123473 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,6 +1,6 @@ from docutils.parsers.rst.roles import set_classes from docutils import nodes -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode from operator import itemgetter @@ -110,24 +110,28 @@ def setup(app): app.connect('doctree-read', collect_scrapy_settings_refs) app.connect('doctree-resolved', replace_settingslist_nodes) + def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/issues/' + text set_classes(options) node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) return [node], [] + def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/commit/' + text set_classes(options) node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) return [node], [] + def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'http://hg.scrapy.org/scrapy/changeset/' + text set_classes(options) diff --git a/docs/conf.py b/docs/conf.py index 640dcd7cb..594740f39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,10 +144,6 @@ html_static_path = ['_static'] # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -html_use_smartypants = True - # Custom sidebar templates, maps document names to template names. #html_sidebars = {} @@ -191,8 +187,8 @@ htmlhelp_basename = 'Scrapydoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', ur'Scrapy Documentation', - ur'Scrapy developers', 'manual'), + ('index', 'Scrapy.tex', u'Scrapy Documentation', + u'Scrapy developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/docs/contributing.rst b/docs/contributing.rst index b0a435ad2..6615840f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,8 @@ Contributing to Scrapy .. important:: - Double check you are reading the most recent version of this document at - http://doc.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document at + https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: @@ -18,13 +18,17 @@ There are many ways to contribute to Scrapy. Here are some of them: * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. -* Submit patches for new functionality and/or bug fixes. Please read - `Writing patches`_ and `Submitting patches`_ below for details on how to +* Submit patches for new functionalities and/or bug fixes. Please read + :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. -* Join the `scrapy-users`_ mailing list and share your ideas on how to +* Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. +* Answer Scrapy questions at + `Stack Overflow `__. + + Reporting bugs ============== @@ -40,13 +44,18 @@ guidelines when reporting a new bug. * check the :ref:`FAQ ` first to see if your issue is addressed in a well-known question -* check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report but check the ticket history and comments, you may - find additional useful information to contribute. +* if you have a general question about scrapy usage, please ask it at + `Stack Overflow `__ + (use "scrapy" tag). -* search the `scrapy-users`_ list to see if it has been discussed there, or - if you're not sure if what you're seeing is a bug. You can also ask in the - `#scrapy` IRC channel. +* check the `open issues`_ to see if the issue has already been reported. If it + has, don't dismiss the report, but check the ticket history and comments. If + you have additional useful information, please leave a comment, or consider + :ref:`sending a pull request ` with a fix. + +* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has + been discussed there, or if you're not sure if what you're seeing is a bug. + You can also ask in the `#scrapy` IRC channel. * write **complete, reproducible, specific bug reports**. The smaller the test case, the better. Remember that other developers won't have your project to @@ -54,17 +63,24 @@ guidelines when reporting a new bug. it. See for example StackOverflow's guide on creating a `Minimal, Complete, and Verifiable example`_ exhibiting the issue. +* the most awesome way to provide a complete reproducible example is to + send a pull request which adds a failing test case to the + Scrapy testing suite (see :ref:`submitting-patches`). + This is helpful even if you don't have an intention to + fix the issue yourselves. + * include the output of ``scrapy version -v`` so developers working on your bug know exactly which version and platform it occurred on, which is often very helpful for reproducing it, or knowing if it was already fixed. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _writing-patches: + Writing patches =============== -The better written a patch is, the higher chance that it'll get accepted and -the sooner that will be merged. +The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: @@ -83,6 +99,8 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +.. _submitting-patches: + Submitting patches ================== @@ -98,16 +116,34 @@ patch, but it's always good to have a patch ready to illustrate your arguments and show that you have put some additional thought into the subject. A good starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea -has been validated and proven useful. Alternatively, you can send an email to -`scrapy-users`_ to discuss your idea first. +has been validated and proven useful. Alternatively, you can start a +conversation in the `Scrapy subreddit`_ to discuss your idea first. + +Sometimes there is an existing pull request for the problem you'd like to +solve, which is stalled for some reason. Often the pull request is in a +right direction, but changes are requested by Scrapy maintainers, and the +original pull request author hasn't had time to address them. +In this case consider picking up this pull request: open +a new pull request with all commits from the original pull request, as well as +additional changes to address the raised issues. Doing so helps a lot; it is +not considered rude as soon as the original author is acknowledged by keeping +his/her commits. + +You can pull an existing pull request to a local branch +by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` +(replace 'upstream' with a remote name for scrapy repository, +``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` +with a name of the branch you want to create locally). +See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally. + When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" -instead of "Fix for #411". -Complete titles make it easy to skim through the issue tracker. +instead of "Fix for #411". Complete titles make it easy to skim through +the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports -removal, etc) in separate commits than functional changes. This will make pull +removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. Coding style @@ -121,30 +157,29 @@ Scrapy: * It's OK to use lines longer than 80 chars if it improves the code readability. -* Don't put your name in the code you contribute. Our policy is to keep - the contributor's name in the `AUTHORS`_ file distributed with Scrapy. - -Scrapy Contrib -============== - -Scrapy contrib shares a similar rationale as Django contrib, which is explained -in `this post `_. If you -are working on a new functionality, please follow that rationale to decide -whether it should be a Scrapy contrib. If unsure, you can ask in -`scrapy-users`_. +* Don't put your name in the code you contribute; git provides enough + metadata to identify author of the code. + See https://help.github.com/articles/setting-your-username-in-git/ for + setup instructions. Documentation policies ====================== * **Don't** use docstrings for documenting classes, or methods which are - already documented in the official (sphinx) documentation. For example, the - :meth:`ItemLoader.add_value` method should be documented in the sphinx - documentation, not its docstring. + already documented in the official (sphinx) documentation. Alternatively, + **do** provide a docstring, but make sure sphinx documentation uses + autodoc_ extension to pull the docstring. For example, the + :meth:`ItemLoader.add_value` method should be either + documented only in the sphinx documentation (not as a docstring), or + it should have a docstring which is pulled to sphinx documentation using + autodoc_ extension. * **Do** use docstrings for documenting functions not present in the official (sphinx) documentation, such as functions from ``scrapy.utils`` package and its sub-modules. +.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html + Tests ===== @@ -197,6 +232,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users +.. _Scrapy subreddit: https://reddit.com/r/scrapy .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests diff --git a/docs/faq.rst b/docs/faq.rst index ad11b071b..7a0628f88 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -21,7 +21,7 @@ Python code. In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like comparing `jinja2`_ to `Django`_. -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _jinja2: http://jinja.pocoo.org/ .. _Django: https://www.djangoproject.com/ @@ -69,12 +69,15 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.3+. +Scrapy is supported under Python 2.7 and Python 3.4+ +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. +PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. .. note:: - Python 3 is not yet supported on Windows. + For Python 3 support on Windows, it is recommended to use + Anaconda/Miniconda as :ref:`outlined in the installation guide `. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/index.rst b/docs/index.rst index 289fb2b1b..7e8c979c4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,14 +13,14 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. -* Ask or search questions in `StackOverflow using the scrapy tag`_, -* Search for information in the `archives of the scrapy-users mailing list`_, or - `post a question`_. +* Ask or search questions in `StackOverflow using the scrapy tag`_. +* Ask or search questions in the `Scrapy subreddit`_. +* Search for questions on the archives of the `scrapy-users mailing list`_. * Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. -.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users -.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 767749ec5..4a9aa3cfb 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,14 +7,26 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above -(except on Windows where Python 3 is not supported yet). +Scrapy runs on Python 2.7 and Python 3.4 or above +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). -If you’re already familiar with installation of Python packages, +If you're using `Anaconda`_ or `Miniconda`_, you can install the package from +the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows +and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c conda-forge scrapy + +Alternatively, if you’re already familiar with installation of Python packages, you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +Note that sometimes this may require solving compilation issues for some Scrapy +dependencies depending on your operating system, so be sure to check the +:ref:`intro-install-platform-notes`. + We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. @@ -96,7 +108,7 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def .. _virtualenv: https://virtualenv.pypa.io .. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/ -.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html +.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html .. _user guide: https://virtualenv.pypa.io/en/stable/userguide/ @@ -105,52 +117,28 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def Platform specific installation notes ==================================== +.. _intro-install-windows: + Windows ------- -* Install Python 2.7 from https://www.python.org/downloads/ +Though it's possible to install Scrapy on Windows using pip, we recommend you +to install `Anaconda`_ or `Miniconda`_ and use the package from the +`conda-forge`_ channel, which will avoid most installation issues. - You need to adjust ``PATH`` environment variable to include paths to - the Python executable and additional scripts. The following paths need to be - added to ``PATH``:: +Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: - C:\Python27\;C:\Python27\Scripts\; + conda install -c conda-forge scrapy - To update the ``PATH`` open a Command prompt and run:: - c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py +.. _intro-install-ubuntu: - Close the command prompt window and reopen it so changes take effect, run the - following command and check it shows the expected Python version:: - - python --version - -* Install `pywin32` from http://sourceforge.net/projects/pywin32/ - - Be sure you download the architecture (win32 or amd64) that matches your system - -* *(Only required for Python<2.7.9)* Install `pip`_ from - https://pip.pypa.io/en/latest/installing/ - - Now open a Command prompt to check ``pip`` is installed correctly:: - - pip --version - -* At this point Python 2.7 and ``pip`` package manager must be working, let's - install Scrapy:: - - pip install Scrapy - -.. note:: - Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support - Python 3 on Windows. - -Ubuntu 12.04 or above +Ubuntu 14.04 or above --------------------- Scrapy is currently tested with recent-enough versions of lxml, twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions. -But it should support older versions of Ubuntu too, like Ubuntu 12.04, +But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are @@ -176,10 +164,12 @@ you can install Scrapy with ``pip`` after that:: pip install scrapy .. note:: - The same non-python dependencies can be used to install Scrapy in Debian - Wheezy (7.0) and above. + The same non-Python dependencies can be used to install Scrapy in Debian + Jessie (8.0) and above. +.. _intro-install-macos: + Mac OS X -------- @@ -199,7 +189,7 @@ solutions: that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: - * Install `homebrew`_ following the instructions in http://brew.sh/ + * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly @@ -234,27 +224,31 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy -Anaconda --------- +PyPy +---- +We recommend using the latest PyPy version. The version tested is 5.9.0. +For PyPy3, only Linux installation was tested. -Using Anaconda is an alternative to using a virtualenv and installing with ``pip``. +Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. +This means that these dependecies will be built during installation. +On OS X, you are likely to face an issue with building Cryptography dependency, +solution to this problem is described +`here `_, +that is to ``brew install openssl`` and then export the flags that this command +recommends (only needed when installing scrapy). Installing on Linux has no special +issues besides installing build dependencies. +Installing scrapy with PyPy on Windows is not tested. -.. note:: +You can check that scrapy is installed correctly by running ``scrapy bench``. +If this command gives errors such as +``TypeError: ... got 2 unexpected keyword arguments``, this means +that setuptools was unable to pick up one PyPy-specific dependency. +To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. - For Windows users, or if you have issues installing through ``pip``, this is - the recommended way to install Scrapy. - -If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ -community have up-to-date packages for Linux, Windows and OS X. - -To install Scrapy using ``conda``, run:: - - conda install -c conda-forge scrapy .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ -.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx .. _lxml: http://lxml.de/ .. _parsel: https://pypi.python.org/pypi/parsel .. _w3lib: https://pypi.python.org/pypi/w3lib @@ -263,9 +257,9 @@ To install Scrapy using ``conda``, run:: .. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ -.. _homebrew: http://brew.sh/ -.. _zsh: http://www.zsh.org/ -.. _Scrapinghub: http://scrapinghub.com -.. _Anaconda: http://docs.continuum.io/anaconda/index -.. _Miniconda: http://conda.pydata.org/docs/install/quick.html -.. _conda-forge: https://conda-forge.github.io/ +.. _homebrew: https://brew.sh/ +.. _zsh: https://www.zsh.org/ +.. _Scrapinghub: https://scrapinghub.com +.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Miniconda: https://conda.io/docs/user-guide/install/index.html +.. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 1da1a4059..6f1c2c43f 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -160,8 +160,8 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: http://scrapy.org/community/ +.. _join the community: https://scrapy.org/community/ .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html .. _Amazon S3: https://aws.amazon.com/s3/ -.. _Sitemaps: http://www.sitemaps.org +.. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3b3bd8d21..20538e90f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -34,7 +34,7 @@ list of Python resources for non-programmers`_. .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial -.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ +.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/ Creating a project @@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file + + middlewares.py # project middlewares file pipelines.py # project pipelines file @@ -452,7 +454,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting its contents. If you run this command twice without removing the file before the second time, you'll end up with a broken JSON file. -You can also used other formats, like `JSON Lines`_:: +You can also use other formats, like `JSON Lines`_:: scrapy crawl quotes -o quotes.jl diff --git a/docs/news.rst b/docs/news.rst index ff1e4ce03..1b8d121a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,13 +3,323 @@ Release notes ============= +Scrapy 1.5.0 (2017-12-29) +------------------------- + +This release brings small new features and improvements across the codebase. +Some highlights: + +* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline. +* Crawling with proxy servers becomes more efficient, as connections + to proxies can be reused now. +* Warnings, exception and logging messages are improved to make debugging + easier. +* ``scrapy parse`` command now allows to set custom request meta via + ``--meta`` argument. +* Compatibility with Python 3.6, PyPy and PyPy3 is improved; + PyPy and PyPy3 are now supported officially, by running tests on CI. +* Better default handling of HTTP 308, 522 and 524 status codes. +* Documentation is improved, as usual. + +Backwards Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Scrapy 1.5 drops support for Python 3.3. +* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +* Logging of settings overridden by ``custom_settings`` is fixed; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're + parsing Scrapy logs, please update your log parsers (:issue:`1343`). +* LinkExtractor now ignores ``m4v`` extension by default, this is change + in behavior. +* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`) + +New features +~~~~~~~~~~~~ + +- Support ```` tags in ``Response.follow`` (:issue:`2785`) +- Support for ``ptpython`` REPL (:issue:`2654`) +- Google Cloud Storage support for FilesPipeline and ImagesPipeline + (:issue:`2923`). +- New ``--meta`` option of the "scrapy parse" command allows to pass additional + request.meta (:issue:`2883`) +- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`) +- Handle HTTP 308 Permanent Redirect (:issue:`2844`) +- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) +- Log versions information at startup (:issue:`2857`) +- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) +- Connections to proxy servers are reused (:issue:`2743`) +- Add template for a downloader middleware (:issue:`2755`) +- Explicit message for NotImplementedError when parse callback not defined + (:issue:`2831`) +- CrawlerProcess got an option to disable installation of root log handler + (:issue:`2921`) +- LinkExtractor now ignores ``m4v`` extension by default +- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and + :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`) +- Show warning when a URL is put to ``Spider.allowed_domains`` instead of + a domain (:issue:`2250`). + +Bug fixes +~~~~~~~~~ + +- Fix logging of settings overridden by ``custom_settings``; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please + update your log parsers if needed (:issue:`1343`) +- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +- Fix PyPy and PyPy3 test failures, support them officially + (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, + :issue:`3048`) +- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) +- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) +- Add verification to check if Request callback is callable (:issue:`2766`) +- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) +- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning + (:issue:`2862`) +- Update deprecated test aliases (:issue:`2876`) +- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) + +Docs +~~~~ + +- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` + setting. (:issue:`2756`) +- Update Contributing docs, document new support channels + (:issue:`2762`, issue:`3038`) +- Include references to Scrapy subreddit in the docs +- Fix broken links; use https:// for external links + (:issue:`2978`, :issue:`2982`, :issue:`2958`) +- Document CloseSpider extension better (:issue:`2759`) +- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example + (:issue:`2781`) +- Spelling mistake and typos + (:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`) +- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) +- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` + (:issue:`2791`) +- Update "Releases" section in README (:issue:`2764`) +- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`) +- Small fix in description of startproject arguments (:issue:`2866`) +- Clarify data types in Response.body docs (:issue:`2922`) +- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`) +- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware + docs (:issue:`2999`) +- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`) +- A better example of ItemExporters usage (:issue:`2989`) +- Document ``from_crawler`` methods for spider and downloader middlewares + (:issue:`3019`) + + +Scrapy 1.4.0 (2017-05-18) +------------------------- + +Scrapy 1.4 does not bring that many breathtaking new features +but quite a few handy improvements nonetheless. + +Scrapy now supports anonymous FTP sessions with customizable user and +password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings. +And if you're using Twisted version 17.1.0 or above, FTP is now available +with Python 3. + +There's a new :meth:`response.follow ` method +for creating requests; **it is now a recommended way to create Requests +in Scrapy spiders**. This method makes it easier to write correct +spiders; ``response.follow`` has several advantages over creating +``scrapy.Request`` objects directly: + +* it handles relative URLs; +* it works properly with non-ascii URLs on non-UTF8 pages; +* in addition to absolute and relative URLs it supports Selectors; + for ```` elements it can also extract their href values. + +For example, instead of this:: + + for href in response.css('li.page a::attr(href)').extract(): + url = response.urljoin(href) + yield scrapy.Request(url, self.parse, encoding=response.encoding) + +One can now write this:: + + for a in response.css('li.page a'): + yield response.follow(a, self.parse) + +Link extractors are also improved. They work similarly to what a regular +modern browser would do: leading and trailing whitespace are removed +from attributes (think ``href=" http://example.com"``) when building +``Link`` objects. This whitespace-stripping also happens for ``action`` +attributes with ``FormRequest``. + +**Please also note that link extractors do not canonicalize URLs by default +anymore.** This was puzzling users every now and then, and it's not what +browsers do in fact, so we removed that extra transformation on extracted +links. + +For those of you wanting more control on the ``Referer:`` header that Scrapy +sends when following links, you can set your own ``Referrer Policy``. +Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and +blindly set it to the URL of the response that generated the HTTP request +(which could leak information on your URL seeds). +By default, Scrapy now behaves much like your regular browser does. +And this policy is fully customizable with W3C standard values +(or with something really custom of your own if you wish). +See :setting:`REFERRER_POLICY` for details. + +To make Scrapy spiders easier to debug, Scrapy logs more stats by default +in 1.4: memory usage stats, detailed retry stats, detailed HTTP error code +stats. A similar change is that HTTP cache path is also visible in logs now. + +Last but not least, Scrapy now has the option to make JSON and XML items +more human-readable, with newlines between items and even custom indenting +offset, using the new :setting:`FEED_EXPORT_INDENT` setting. + +Enjoy! (Or read on for the rest of changes in this release.) + +Deprecations and Backwards Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` + (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): + **warning, this is technically backwards-incompatible** +- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`); + **this is technically backwards-incompatible** so please check if you have + any non-default ``MEMUSAGE_***`` options set. +- ``EDITOR`` environment variable now takes precedence over ``EDITOR`` + option defined in settings.py (:issue:`1829`); Scrapy default settings + no longer depend on environment variables. **This is technically a backwards + incompatible change**. +- ``Spider.make_requests_from_url`` is deprecated + (:issue:`1728`, fixes :issue:`1495`). + +New Features +~~~~~~~~~~~~ + +- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) +- Support `brotli`_-compressed content; requires optional `brotlipy`_ + (:issue:`2535`) +- New :ref:`response.follow ` shortcut + for creating requests (:issue:`1940`) +- Added ``flags`` argument and attribute to :class:`Request ` + objects (:issue:`2047`) +- Support Anonymous FTP (:issue:`2342`) +- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/`` + stats to :class:`RetryMiddleware ` + (:issue:`2543`) +- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/`` + stats to :class:`HttpErrorMiddleware ` + (:issue:`2566`) +- Customizable :setting:`Referrer policy ` in + :class:`RefererMiddleware ` + (:issue:`2306`) +- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`) +- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`) +- Warn users when project contains duplicate spider names (fixes :issue:`2181`) +- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`) +- :ref:`Media downloads `, with :class:`FilesPipelines` + or :class:`ImagesPipelines`, can now optionally handle HTTP redirects + using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`) +- Accept non-complete responses from websites using a new + :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`) +- Optional pretty-printing of JSON and XML items via + :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`) +- Allow dropping fields in ``FormRequest.from_response`` formdata when + ``None`` value is passed (:issue:`667`) +- Per-request retry times with the new :reqmeta:`max_retry_times` meta key + (:issue:`2642`) +- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command + (:issue:`2740`) + +.. _brotli: https://github.com/google/brotli +.. _brotlipy: https://github.com/python-hyper/brotlipy/ + +Bug fixes +~~~~~~~~~ + +- LinkExtractor now strips leading and trailing whitespaces from attributes + (:issue:`2547`, fixes :issue:`1614`) +- Properly handle whitespaces in action attribute in :class:`FormRequest` + (:issue:`2548`) +- Buffer CONNECT response bytes from proxy until all HTTP headers are received + (:issue:`2495`, fixes :issue:`2491`) +- FTP downloader now works on Python 3, provided you use Twisted>=17.1 + (:issue:`2599`) +- Use body to choose response type after decompressing content (:issue:`2393`, + fixes :issue:`2145`) +- Always decompress ``Content-Encoding: gzip`` at :class:`HttpCompressionMiddleware + ` stage (:issue:`2391`) +- Respect custom log level in ``Spider.custom_settings`` (:issue:`2581`, + fixes :issue:`1612`) +- 'make htmlview' fix for macOS (:issue:`2661`) +- Remove "commands" from the command list (:issue:`2695`) +- Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`) +- Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`) +- ImagesPipeline: fixed processing of transparent PNG images with palette + (:issue:`2675`) + +Cleanups & Refactoring +~~~~~~~~~~~~~~~~~~~~~~ + +- Tests: remove temp files and folders (:issue:`2570`), + fixed ProjectUtilsTest on OS X (:issue:`2569`), + use portable pypy for Linux on Travis CI (:issue:`2710`) +- Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`) +- Remove “Python 3 progress” badge (:issue:`2567`) +- Add a couple more lines to ``.gitignore`` (:issue:`2557`) +- Remove bumpversion prerelease configuration (:issue:`2159`) +- Add codecov.yml file (:issue:`2750`) +- Set context factory implementation based on Twisted version (:issue:`2577`, + fixes :issue:`2560`) +- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`) +- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`) +- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`) +- Change "localhost" test server certificate (:issue:`2720`) +- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`) + +Documentation +~~~~~~~~~~~~~ + +- Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) +- Mention issue with :meth:`FormRequest.from_response + ` due to bug in lxml (:issue:`2572`) +- Use single quotes uniformly in templates (:issue:`2596`) +- Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) +- Removed section on deprecated ``contrib/`` (:issue:`2636`) +- Recommend Anaconda when installing Scrapy on Windows + (:issue:`2477`, fixes :issue:`2475`) +- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`) +- Rearrange selector sections (:issue:`2705`) +- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`) +- Mention how to disable request filtering in documentation of + :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`) +- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`) +- Open file in text mode in JSON item writer example (:issue:`2729`) +- Clarify ``allowed_domains`` example (:issue:`2670`) + + +Scrapy 1.3.3 (2017-03-10) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing + dependencies and wrong :setting:`SPIDER_MODULES`. + These exceptions were silenced as warnings since 1.3.0. + A new setting is introduced to toggle between warning or exception if needed ; + see :setting:`SPIDER_LOADER_WARN_ONLY` for details. + + Scrapy 1.3.2 (2017-02-13) ------------------------- Bug fixes ~~~~~~~~~ -- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`). - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) @@ -101,6 +411,12 @@ Dependencies & Cleanups downloader middlewares. +Scrapy 1.2.3 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -203,7 +519,7 @@ Refactoring - ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`). -.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url +.. _w3lib.url: https://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url Tests & Requirements ~~~~~~~~~~~~~~~~~~~~ @@ -229,6 +545,12 @@ Documentation - Add StackOverflow as a support channel (:issue:`2257`). +Scrapy 1.1.4 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -501,6 +823,12 @@ Bugfixes to same remote host (:issue:`1912`). +Scrapy 1.0.7 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.0.6 (2016-05-04) ------------------------- diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..8e7611d21 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx>=1.6 +sphinx_rtd_theme \ No newline at end of file diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index b83946a58..c9bece753 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -88,6 +88,7 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_ENABLED` * :setting:`AUTOTHROTTLE_START_DELAY` * :setting:`AUTOTHROTTLE_MAX_DELAY` +* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` * :setting:`CONCURRENT_REQUESTS_PER_IP` diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 28ed7c064..eb02086dc 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -20,7 +20,7 @@ These are some common properties often found in broad crawls: * they crawl many domains (often, unbounded) instead of a specific set of sites -* they don't necessarily crawl domains to completion, because it would +* they don't necessarily crawl domains to completion, because it would be impractical (or impossible) to do so, and instead limit the crawl by time or number of pages crawled @@ -85,8 +85,8 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may fine -though. +production. Using ``DEBUG`` level when developing your (broad) crawler may be +fine though. To set the log level use:: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index eaeeee113..3088017cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -55,6 +55,7 @@ structure by default, similar to this:: myproject/ __init__.py items.py + middlewares.py pipelines.py settings.py spiders/ @@ -187,7 +188,7 @@ startproject Creates a new Scrapy project named ``project_name``, under the ``project_dir`` directory. -If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``. +If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``. Usage example:: @@ -291,12 +292,12 @@ edit * Syntax: ``scrapy edit `` * Requires project: *yes* -Edit the given spider using the editor defined in the :setting:`EDITOR` -setting. +Edit the given spider using the editor defined in the ``EDITOR`` environment +variable or (if unset) the :setting:`EDITOR` setting. This command is provided only as a convenience shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and -debug his spiders. +debug spiders. Usage example:: @@ -430,6 +431,9 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback + request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` @@ -543,7 +547,7 @@ Example:: COMMANDS_MODULE = 'mybot.commands' -.. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html +.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points ------------------------------------------- diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index a3e72097c..d1991c02f 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -142,4 +142,4 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. -.. _base tag: http://www.w3schools.com/tags/tag_base.asp +.. _base tag: https://www.w3schools.com/tags/tag_base.asp diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index bc48ddce7..361914a29 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -50,10 +50,10 @@ them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. .. _Scrapyd: https://github.com/scrapy/scrapyd -.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html -.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/ +.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html +.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: http://doc.scrapinghub.com/shub.html -.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html -.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: http://scrapinghub.com/ +.. _shub: https://doc.scrapinghub.com/shub.html +.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html +.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html +.. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0ef3fb071..dfe4c13b4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -157,6 +157,17 @@ more of the following methods: :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -226,6 +237,15 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. + +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. + .. setting:: COOKIES_DEBUG COOKIES_DEBUG @@ -645,6 +665,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. + This middleware also supports decoding `brotli-compressed`_ responses, + provided `brotlipy`_ is installed. + +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotlipy: https://pypi.python.org/pypi/brotlipy + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -846,6 +872,11 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. +Maximum number of retries can also be specified per-request using +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +When initialized, the :reqmeta:`max_retry_times` meta key takes higher +precedence over the :setting:`RETRY_TIMES` setting. + .. setting:: RETRY_HTTP_CODES RETRY_HTTP_CODES diff --git a/docs/topics/email.rst b/docs/topics/email.rst index aac93a91a..949cdc638 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` setting will be used. If not given, no SMTP authentication will be performed. - :type smtphost: str + :type smtphost: str or bytes :param smtppass: the SMTP pass for authentication. - :type smtppass: str + :type smtppass: str or bytes :param smtpport: the SMTP port to connect to :type smtpport: int diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index cc02369d4..09cb8ed66 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -39,6 +39,14 @@ For example:: if 'Bandwidth exceeded' in response.body: raise CloseSpider('bandwidth_exceeded') +DontCloseSpider +--------------- + +.. exception:: DontCloseSpider + +This exception can be raised in a :signal:`spider_idle` signal handler to +prevent the spider from being closed. + IgnoreRequest ------------- diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 85c73222d..95f7920f8 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -36,38 +36,36 @@ to export 3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal the end of the exporting process -Here you can see an :doc:`Item Pipeline ` which uses an Item -Exporter to export scraped items to different files, one per spider:: +Here you can see an :doc:`Item Pipeline ` which uses multiple +Item Exporters to group scraped items to different files according to the +value of one of their fields:: - from scrapy import signals - from scrapy.exporters import XmlItemExporter + from scrapy.exporters import XmlItemExporter - class XmlExportPipeline(object): + class PerYearXmlExportPipeline(object): + """Distribute items across multiple XML files according to their 'year' field""" - def __init__(self): - self.files = {} + def open_spider(self, spider): + self.year_to_exporter = {} - @classmethod - def from_crawler(cls, crawler): - pipeline = cls() - crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) - crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) - return pipeline + def close_spider(self, spider): + for exporter in self.year_to_exporter.values(): + exporter.finish_exporting() + exporter.file.close() - def spider_opened(self, spider): - file = open('%s_products.xml' % spider.name, 'w+b') - self.files[spider] = file - self.exporter = XmlItemExporter(file) - self.exporter.start_exporting() + def _exporter_for_item(self, item): + year = item['year'] + if year not in self.year_to_exporter: + f = open('{}.xml'.format(year), 'wb') + exporter = XmlItemExporter(f) + exporter.start_exporting() + self.year_to_exporter[year] = exporter + return self.year_to_exporter[year] - def spider_closed(self, spider): - self.exporter.finish_exporting() - file = self.files.pop(spider) - file.close() - - def process_item(self, item, spider): - self.exporter.export_item(item) - return item + def process_item(self, item, spider): + exporter = self._exporter_for_item(item) + exporter.export_item(item) + return item .. _topics-exporters-field-serialization: @@ -140,7 +138,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8') +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -149,7 +147,7 @@ BaseItemExporter These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, - :attr:`export_empty_fields`, :attr:`encoding`. + :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. .. method:: export_item(item) @@ -216,6 +214,15 @@ BaseItemExporter encoding). Other value types are passed unchanged to the specific serialization library. + .. attribute:: indent + + Amount of spaces used to indent the output on each level. Defaults to ``0``. + + * ``indent=None`` selects the most compact representation, + all items in the same line with no indentation + * ``indent<=0`` each item on its own line, no indentation + * ``indent>0`` each item on its own line, indented with the provided numeric value + .. highlight:: none XmlItemExporter diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7f2952f4c..c421a5e05 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -220,7 +220,6 @@ can be configured with the following settings: * :setting:`MEMUSAGE_LIMIT_MB` * :setting:`MEMUSAGE_WARNING_MB` * :setting:`MEMUSAGE_NOTIFY_MAIL` -* :setting:`MEMUSAGE_REPORT` * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension @@ -278,9 +277,11 @@ CLOSESPIDER_ITEMCOUNT Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than -that amount if items and those items are passed by the item pipeline, the -spider will be closed with the reason ``closespider_itemcount``. If zero (or -non set), spiders won't be closed by number of passed items. +that amount and those items are passed by the item pipeline, the +spider will be closed with the reason ``closespider_itemcount``. +Requests which are currently in the downloader queue (up to +:setting:`CONCURRENT_REQUESTS` requests) are still processed. +If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT @@ -372,4 +373,4 @@ For more info see `Debugging in Python`. This extension only works on POSIX-compliant platforms (ie. not Windows). .. _Python debugger: https://docs.python.org/2/library/pdb.html -.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html +.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efdd8c46b..135d05c93 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` + * :setting:`FEED_EXPORT_INDENT` .. currentmodule:: scrapy.extensions.feedexport @@ -266,6 +267,22 @@ If an exporter requires a fixed set of fields (this is the case for is empty or None, then Scrapy tries to infer field names from the exported data - currently it uses field names from the first item. +.. setting:: FEED_EXPORT_INDENT + +FEED_EXPORT_INDENT +------------------ + +Default: ``0`` + +Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` +is a non-negative integer, then array elements and object members will be pretty-printed +with that indent level. An indent level of ``0`` (the default), or negative, +will put each item on a new line. ``None`` selects the most compact representation. + +Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter` +and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting +to ``.json`` or ``.xml``. + .. setting:: FEED_STORE_EMPTY FEED_STORE_EMPTY diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 8f0a5767b..4ea8d3bd0 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the Project`_ used in the :ref:`tutorial ` but with a different face. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Google Directory: http://directory.google.com/ .. _Open Directory Project: http://www.dmoz.org diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst index 0cf45861a..2c85848be 100644 --- a/docs/topics/firefox.rst +++ b/docs/topics/firefox.rst @@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one after applying some browser clean up and executing Javascript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. +extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things when working with Firefox and XPath: @@ -71,11 +71,11 @@ Firecookie `Firecookie`_ makes it easier to view and manage cookies. You can use this extension to create a new cookie, delete existing cookies, see a list of cookies -for the current site, manage cookies permissions and a lot more. +for the current site, manage cookies permissions and a lot more. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA -.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ +.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ .. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/ .. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/ .. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 8c7aa361f..38265b474 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -107,7 +107,7 @@ format:: class JsonWriterPipeline(object): def open_spider(self, spider): - self.file = open('items.jl', 'wb') + self.file = open('items.jl', 'w') def close_spider(self, spider): self.file.close() @@ -134,7 +134,7 @@ method and how to clean up the resources properly.:: import pymongo class MongoPipeline(object): - + collection_name = 'scrapy_items' def __init__(self, mongo_uri, mongo_db): @@ -156,7 +156,7 @@ method and how to clean up the resources properly.:: self.client.close() def process_item(self, item, spider): - self.db[self.collection_name].insert(dict(item)) + self.db[self.collection_name].insert_one(dict(item)) return item .. _MongoDB: https://www.mongodb.org/ @@ -208,7 +208,7 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite item["screenshot_filename"] = filename return item -.. _Splash: http://splash.readthedocs.io/en/stable/ +.. _Splash: https://splash.readthedocs.io/en/stable/ .. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html Duplicates filter @@ -248,4 +248,3 @@ To activate an Item Pipeline component you must add its class to the The integer values you assign to classes in this setting determine the order in which they run: items go through from lower valued to higher valued classes. It's customary to define these numbers in the 0-1000 range. - diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 4a8f47e93..4423bbda2 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -21,7 +21,7 @@ their available fields. Various Scrapy components use extra information provided by Items: exporters look at declared fields to figure out columns to export, serialization can be customized using Item fields metadata, :mod:`trackref` -tracks Item instances to help finding memory leaks +tracks Item instances to help find memory leaks (see :ref:`topics-leaks-trackrefs`), etc. .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 4f9e38086..06c7fff3d 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. -.. _pickle: http://docs.python.org/library/pickle.html +.. _pickle: https://docs.python.org/library/pickle.html diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 92590c180..af14d14e8 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. +If you're using Python3, see :ref:`topics-leaks-muppy`. .. _Guppy library: https://pypi.python.org/pypi/guppy @@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the .. _Guppy documentation: http://guppy-pe.sourceforge.net/ +.. _topics-leaks-muppy: + +Debugging memory leaks with muppy +================================= +If you're using Python 3, you can use muppy from `Pympler`_. + +.. _Pympler: https://pypi.org/project/Pympler/ + +If you use ``pip``, you can install muppy with the following command:: + + pip install Pympler + +Here's an example to view all Python objects available in +the heap using muppy:: + + >>> from pympler import muppy + >>> all_objects = muppy.get_objects() + >>> len(all_objects) + 28667 + >>> from pympler import summary + >>> suml = summary.summarize(all_objects) + >>> summary.print_(suml) + types | # objects | total size + ==================================== | =========== | ============ + ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== @@ -518,8 +533,8 @@ a footer of a page that looks something like: Example:: diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index ac3b614fc..0986929ad 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -27,7 +27,7 @@ Scrapy from scripts as described in :ref:`run-from-script`. Log levels ========== -Python's builtin logging defines 5 different levels to indicate severity on a +Python's builtin logging defines 5 different levels to indicate the severity of a given log message. Here are the standard ones, listed in decreasing order: 1. ``logging.CRITICAL`` - for critical errors (highest severity) @@ -47,20 +47,20 @@ level:: There are shortcuts for issuing log messages on any of the standard 5 levels, and there's also a general ``logging.log`` method which takes a given level as -argument. If you need so, last example could be rewrote as:: +argument. If needed, the last example could be rewritten as:: import logging logging.log(logging.WARNING, "This is a warning") -On top of that, you can create different "loggers" to encapsulate messages (For -example, a common practice it's to create different loggers for every module). +On top of that, you can create different "loggers" to encapsulate messages. (For +example, a common practice is to create different loggers for every module). These loggers can be configured independently, and they allow hierarchical constructions. -Last examples use the root logger behind the scenes, which is a top level +The previous examples use the root logger behind the scenes, which is a top level logger where all messages are propagated to (unless otherwise specified). Using ``logging`` helpers is merely a shortcut for getting the root logger -explicitly, so this is also an equivalent of last snippets:: +explicitly, so this is also an equivalent of the last snippets:: import logging logger = logging.getLogger() @@ -95,14 +95,14 @@ Logging from Spiders ==================== Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider -instance, that can be accessed and used like this:: +instance, which can be accessed and used like this:: import scrapy class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -118,7 +118,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket) +* Specifying where to store the media (filesystem directory, Amazon S3 bucket, + Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: @@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: Supported Storage ================= -File system is currently the only officially supported storage, but there is -also support for storing files in `Amazon S3`_. +File system is currently the only officially supported storage, but there are +also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_. .. _Amazon S3: https://aws.amazon.com/s3/ +.. _Google Cloud Storage: https://cloud.google.com/storage/ File system storage ------------------- @@ -169,7 +171,53 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like +self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: + + AWS_ENDPOINT_URL = 'http://minio.example.com:9000' + +For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:: + + AWS_USE_SSL = False # or True (None by default) + AWS_VERIFY = False # or True (None by default) + +.. _Minio: https://github.com/minio/minio +.. _s3.scality: https://s3.scality.com/ +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + +Google Cloud Storage +--------------------- + +.. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL + +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage +bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + +For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:: + + IMAGES_STORE = 'gs://bucket/images/' + GCS_PROJECT_ID = 'project_id' + +For information about authentication, see this `documentation`_. + +.. _documentation: https://cloud.google.com/docs/authentication/production + +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl Usage example ============= @@ -320,6 +368,18 @@ all be dropped because at least one dimension is shorter than the constraint. By default, there are no size constraints, so all images are processed. +Allowing redirections +--------------------- + +.. setting:: MEDIA_ALLOW_REDIRECTS + +By default media pipelines ignore redirects, i.e. an HTTP redirection +to a media file URL request will mean the media download is considered failed. + +To handle media redirections, set this setting to ``True``:: + + MEDIA_ALLOW_REDIRECTS = True + .. _topics-media-pipeline-override: Extending the Media Pipelines diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 25ae4b5ba..02cfa9b05 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -238,7 +238,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is @@ -248,10 +248,10 @@ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. .. _Tor project: https://www.torproject.org/ -.. _commercial support: http://scrapy.org/support/ -.. _ProxyMesh: http://proxymesh.com/ +.. _commercial support: https://scrapy.org/support/ +.. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html -.. _Crawlera: http://scrapinghub.com/crawlera -.. _scrapoxy: http://scrapoxy.io/ +.. _Crawlera: https://scrapinghub.com/crawlera +.. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9a6e0d1b6..e29914dbf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -80,6 +80,8 @@ Request objects attributes of the cookie. This is only useful if the cookies are saved for later requests. + .. reqmeta:: dont_merge_cookies + When some site returns cookies (in a response) those are stored in the cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. However, if, for some @@ -294,7 +296,7 @@ Those are: * :reqmeta:`dont_retry` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`handle_httpstatus_all` -* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor) +* :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` * :reqmeta:`redirect_urls` @@ -303,7 +305,12 @@ Those are: * :reqmeta:`download_timeout` * :reqmeta:`download_maxsize` * :reqmeta:`download_latency` +* :reqmeta:`download_fail_on_dataloss` * :reqmeta:`proxy` +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`referrer_policy` +* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress @@ -330,6 +337,23 @@ started, i.e. HTTP message sent over the network. This meta key only becomes available when the response has been downloaded. While most other meta keys are used to control Scrapy behavior, this one is supposed to be read-only. +.. reqmeta:: download_fail_on_dataloss + +download_fail_on_dataloss +------------------------- + +Whether or not to fail on broken responses. See: +:setting:`DOWNLOAD_FAIL_ON_DATALOSS`. + +.. reqmeta:: max_retry_times + +max_retry_times +--------------- + +The meta key is used set retry times per request. When initialized, the +:reqmeta:`max_retry_times` meta key takes higher precedence over the +:setting:`RETRY_TIMES` setting. + .. _topics-request-response-ref-request-subclasses: Request subclasses @@ -405,7 +429,9 @@ fields with form data from :class:`Response` objects. :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is - overridden by the one passed in this parameter. + overridden by the one passed in this parameter. If a value passed in + this parameter is ``None``, the field will not be included in the + request, even if it was present in the response ```` element. :type formdata: dict :param clickdata: attributes to lookup the control clicked. If it's not @@ -501,11 +527,11 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. It must be str, not unicode, unless you're - using a encoding-aware :ref:`Response subclass - `, such as - :class:`TextResponse`. - :type body: str + :param body: the response body. To access the decoded text as str (unicode + in Python 2) you can use ``response.text`` from an encoding-aware + :ref:`Response subclass `, + such as :class:`TextResponse`. + :type body: bytes :param flags: is a list containing the initial values for the :attr:`Response.flags` attribute. If given, the list will be shallow @@ -710,7 +736,7 @@ HtmlResponse objects which adds encoding auto-discovering support by looking into the HTML `meta http-equiv`_ attribute. See :attr:`TextResponse.encoding`. -.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp +.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp XmlResponse objects ------------------- diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index 57921b901..a3d6f7698 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -10,4 +10,4 @@ Scrapyd has been moved into a separate project. Its documentation is now hosted at: - http://scrapyd.readthedocs.org/en/latest/ + https://scrapyd.readthedocs.io/en/latest/ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8a5d44aac..8ac40c3cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -36,7 +36,7 @@ documents. For a complete reference of the selectors API see :ref:`Selector reference ` -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html .. _cssselect: https://pypi.python.org/pypi/cssselect/ @@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy documentation server: - http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://doc.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -99,7 +99,7 @@ Here's its HTML code: First, let's open the shell:: - scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. @@ -593,6 +593,9 @@ Built-in Selectors reference .. module:: scrapy.selector :synopsis: Selector class +Selector objects +---------------- + .. class:: Selector(response=None, text=None, type=None) An instance of :class:`Selector` is a wrapper over response to select @@ -714,13 +717,9 @@ SelectorList objects Call the ``.re()`` method for each element in this list and return their results flattened, as a list of unicode strings. - .. method:: __nonzero__() - - returns True if the list is not empty, False otherwise. - Selector examples on HTML response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Here's a couple of :class:`Selector` examples to illustrate several concepts. In all cases, we assume there is already a :class:`Selector` instantiated with @@ -745,7 +744,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: print node.xpath("@class").extract() Selector examples on XML response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- Here's a couple of examples to illustrate several concepts. In both cases we assume there is already a :class:`Selector` instantiated with an @@ -767,7 +766,7 @@ assume there is already a :class:`Selector` instantiated with an .. _removing-namespaces: Removing namespaces -~~~~~~~~~~~~~~~~~~~ +------------------- When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f616742c4..076dc6bfd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend `. .. setting:: BOT_NAME +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. + +.. setting:: AWS_ENDPOINT_URL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. +By default SSL will be used. + +.. setting:: AWS_USE_SSL + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. + +.. setting:: AWS_VERIFY + BOT_NAME -------- @@ -604,6 +632,32 @@ If you want to disable it set to 0. This feature needs Twisted >= 11.1. +.. setting:: DOWNLOAD_FAIL_ON_DATALOSS + +DOWNLOAD_FAIL_ON_DATALOSS +------------------------- + +Default: ``True`` + +Whether or not to fail on broken responses, that is, declared +``Content-Length`` does not match content sent by the server or chunked +response was not properly finish. If ``True``, these responses raise a +``ResponseFailed([_DataLoss])`` error. If ``False``, these responses +are passed through and the flag ``dataloss`` is added to the response, i.e.: +``'dataloss' in response.flags`` is ``True``. + +Optionally, this can be set per-request basis by using the +:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. + +.. note:: + + A broken response, or data loss error, may happen under several + circumstances, from server misconfiguration to network errors to data + corruption. It is up to the user to decide if it makes sense to process + broken responses considering they may contain partial or incomplete content. + If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -620,6 +674,13 @@ override its ``request_fingerprint`` method. This method should accept scrapy :class:`~scrapy.http.Request` object and return its fingerprint (a string). +You can disable filtering of duplicate requests by setting +:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. +Be very careful about this however, because you can get into crawling loops. +It's usually a better idea to set the ``dont_filter`` parameter to +``True`` on the specific :class:`~scrapy.http.Request` that should not be +filtered. + .. setting:: DUPEFILTER_DEBUG DUPEFILTER_DEBUG @@ -635,11 +696,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r EDITOR ------ -Default: `depends on the environment` +Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows) -The editor to use for editing spiders with the :command:`edit` command. It -defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults -to ``vi`` (on Unix systems) or the IDLE editor (on Windows). +The editor to use for editing spiders with the :command:`edit` command. +Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit` +command will prefer it over the default setting. .. setting:: EXTENSIONS @@ -926,19 +987,6 @@ Example:: See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_REPORT - -MEMUSAGE_REPORT ---------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -Whether to send a memory usage report after each spider has been closed. - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -982,7 +1030,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option. If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. -.. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. _wget: https://www.gnu.org/software/wget/manual/wget.html .. setting:: REACTOR_THREADPOOL_MAXSIZE @@ -1154,6 +1202,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. setting:: SPIDER_LOADER_WARN_ONLY + +SPIDER_LOADER_WARN_ONLY +----------------------- + +.. versionadded:: 1.3.3 + +Default: ``False`` + +By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +it will fail loudly if there is any ``ImportError`` exception. +But you can choose to silence this exception and turn it into a simple +warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. + +.. note:: + Some :ref:`scrapy commands ` run with this setting to ``True`` + already (i.e. they will only issue a warning and will not fail) + since they do not actually need to load spider classes to work: + :command:`scrapy runspider `, + :command:`scrapy settings `, + :command:`scrapy startproject `, + :command:`scrapy version `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES @@ -1274,14 +1345,14 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html .. setting:: USER_AGENT USER_AGENT ---------- -Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index ef6aeeed3..11ab199f2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: [settings] shell = bpython -.. _IPython: http://ipython.org/ -.. _IPython installation guide: http://ipython.org/install.html -.. _bpython: http://www.bpython-interpreter.org/ +.. _IPython: https://ipython.org/ +.. _IPython installation guide: https://ipython.org/install.html +.. _bpython: https://www.bpython-interpreter.org/ Launch the shell ================ @@ -142,7 +142,7 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then proceed to scrape the https://reddit.com +https://scrapy.org page, and then proceed to scrape the https://reddit.com page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. @@ -154,7 +154,7 @@ shell works. First, we launch the shell:: - scrapy shell 'http://scrapy.org' --nolog + scrapy shell 'https://scrapy.org' --nolog Then, the shell fetches the URL (using the Scrapy downloader) and prints the list of available objects and useful shortcuts (you'll notice that these lines @@ -164,7 +164,7 @@ all start with the ``[s]`` prefix):: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request + [s] request [s] response <200 https://scrapy.org/> [s] settings [s] spider @@ -182,7 +182,7 @@ After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("http://reddit.com") + >>> fetch("https://reddit.com") >>> response.xpath('//title/text()').extract() ['reddit: the front page of the internet'] diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 0306ee4a5..cf1588df8 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -189,14 +189,20 @@ spider_idle the engine starts closing the spider. After the spider has finished closing, the :signal:`spider_closed` signal is sent. - You can, for example, schedule some requests in your :signal:`spider_idle` - handler to prevent the spider from being closed. + You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to + prevent the spider from being closed. This signal does not support returning deferreds from their handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: Scheduling some requests in your :signal:`spider_idle` handler does + **not** guarantee that it can prevent the spider from being closed, + although it sometimes can. That's because the spider may still remain idle + if all the scheduled requests are rejected by the scheduler (e.g. filtered + due to duplication). + spider_error ------------ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fc7669437..0af26be73 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -95,7 +95,7 @@ following methods: it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` + :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. :param response: the response which generated this output from the @@ -166,6 +166,17 @@ following methods: :param spider: the spider to whom the start requests belong :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception @@ -190,9 +201,13 @@ DepthMiddleware .. class:: DepthMiddleware - DepthMiddleware is a scrape middleware used for tracking the depth of each - Request inside the site being scraped. It can be used to limit the maximum - depth to scrape or things like that. + DepthMiddleware is used for tracking the depth of each Request inside the + site being scraped. It works by setting `request.meta['depth'] = 0` whenever + there is no value previously set (usually just the first Request) and + incrementing it by 1 otherwise. + + It can be used to limit the maximum depth to scrape, control Request + priority based on their depth, and things like that. The :class:`DepthMiddleware` can be configured through the following settings (see the settings documentation for more info): @@ -330,6 +345,90 @@ Default: ``True`` Whether to enable referer middleware. +.. setting:: REFERRER_POLICY + +REFERRER_POLICY +^^^^^^^^^^^^^^^ + +.. versionadded:: 1.4 + +Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` + +.. reqmeta:: referrer_policy + +`Referrer Policy`_ to apply when populating Request "Referer" header. + +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERRER_POLICY`` setting. + +Acceptable values for REFERRER_POLICY +************************************* + +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. + +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== + +.. autoclass:: DefaultReferrerPolicy +.. warning:: + Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers — will send a non-empty + "Referer" header from any ``http(s)://`` to any ``https://`` URL, + even if the domain is different. + + `"same-origin"`_ may be a better choice if you want to remove referrer + information for cross-domain requests. + +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy +.. note:: + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. + +.. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer +.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade +.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin +.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin +.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin +.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin +.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin +.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + UrlLengthMiddleware ------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5e69055d1..697732b47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -81,11 +81,14 @@ scrapy.Spider specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + Let's say your target url is ``https://www.example.com/1.html``, + then add ``'example.com'`` to the list. + .. attribute:: start_urls A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data + listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings @@ -575,8 +578,7 @@ CSVFeedSpider .. attribute:: headers - A list of the rows contained in the file CSV feed which will be used to - extract fields from it. + A list of the column names in the CSV file. .. method:: parse_row(response, row) @@ -749,8 +751,8 @@ Combine SitemapSpider with other sources of urls:: def parse_other(self, response): pass # ... scrape other here ... -.. _Sitemaps: http://www.sitemaps.org -.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index +.. _Sitemaps: https://www.sitemaps.org/index.html +.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: http://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain -.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/ +.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 679bb56ff..81ce800aa 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -37,5 +37,5 @@ To use the packages: .. warning:: `python-scrapy` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. -.. _Scrapinghub: http://scrapinghub.com/ +.. _Scrapinghub: https://scrapinghub.com/ .. _GitHub repo: https://github.com/scrapy/scrapy diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 40316968f..6290adbe2 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -20,7 +20,7 @@ _filename = None _contents = None # A regex that matches standard linkcheck output lines -line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh index dc20e16e4..842d0e46e 100755 --- a/extras/coverage-report.sh +++ b/extras/coverage-report.sh @@ -1,6 +1,6 @@ # Run tests, generate coverage report and open it on a browser # -# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage +# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage coverage run --branch $(which trial) --reporter=text tests coverage html -i diff --git a/extras/qpsclient.py b/extras/qpsclient.py index bb83588dd..7554f7eec 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -41,7 +41,7 @@ class QPSSpider(Spider): slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)] + urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] else: urls = [url] diff --git a/requirements-py3.txt b/requirements-py3.txt index cc0a7f644..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 15.5.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 diff --git a/requirements.txt b/requirements.txt index 392f83dd6..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=1.1 +parsel>=1.4 diff --git a/scrapy/VERSION b/scrapy/VERSION index 1892b9267..bc80560fa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.2 +1.5.0 diff --git a/scrapy/__main__.py b/scrapy/__main__.py new file mode 100644 index 000000000..e467e057f --- /dev/null +++ b/scrapy/__main__.py @@ -0,0 +1,4 @@ +from scrapy.cmdline import execute + +if __name__ == '__main__': + execute() diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 60e0de1f2..f55ecc213 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -4,12 +4,12 @@ from six.moves import copyreg if sys.version_info[0] == 2: from urlparse import urlparse - # workaround for http://bugs.python.org/issue7904 - Python < 2.7 + # workaround for https://bugs.python.org/issue7904 - Python < 2.7 if urlparse('s3://bucket/key').netloc != 'bucket': from urlparse import uses_netloc uses_netloc.append('s3') - # workaround for http://bugs.python.org/issue9374 - Python < 2.7.4 + # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 if urlparse('s3://bucket/key?key=value').query != 'key=value': from urlparse import uses_query uses_query.append('s3') diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cb7bbd64d..dc6b59fe0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,5 +1,5 @@ from __future__ import print_function -import sys +import sys, os import optparse import cProfile import inspect @@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings +from scrapy.utils.python import garbage_collect from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): @@ -20,7 +21,8 @@ def _iter_command_classes(module_name): for obj in vars(module).values(): if inspect.isclass(obj) and \ issubclass(obj, ScrapyCommand) and \ - obj.__module__ == module.__name__: + obj.__module__ == module.__name__ and \ + not obj == ScrapyCommand: yield obj def _get_commands_from_module(module, inproject): @@ -106,6 +108,12 @@ def execute(argv=None, settings=None): if settings is None: settings = get_project_settings() + # set EDITOR from environment if available + try: + editor = os.environ['EDITOR'] + except KeyError: pass + else: + settings['EDITOR'] = editor check_deprecated_settings(settings) # --- backwards compatibility for scrapy.conf.settings singleton --- @@ -158,4 +166,9 @@ def _run_command_profiled(cmd, args, opts): p.dump_stats(opts.profile) if __name__ == '__main__': - execute() + try: + execute() + finally: + # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() + # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies + garbage_collect() diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b986bf9d..8093fd402 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -56,3 +56,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 2df6a730c..25d843a53 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,8 +1,10 @@ -import sys, os +import sys +import os from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError + class Command(ScrapyCommand): requires_project = True @@ -15,7 +17,8 @@ class Command(ScrapyCommand): return "Edit spider" def long_desc(self): - return "Edit a spider using the editor defined in EDITOR setting" + return ("Edit a spider using the editor defined in the EDITOR environment" + " variable or else the EDITOR setting") def _err(self, msg): sys.stderr.write(msg + os.linesep) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 5264982b6..69418a478 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,5 @@ from __future__ import print_function +import json import logging from w3lib.url import is_url @@ -48,6 +49,8 @@ class Command(ScrapyCommand): help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", help="use this callback for parsing, instead looking for a callback") + parser.add_option("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", @@ -142,7 +145,8 @@ class Command(ScrapyCommand): logger.error('Unable to find spider for: %(url)s', {'url': url}) - request = Request(url, opts.callback) + # Request requires callback argument as callable or None, not string + request = Request(url, None) _start_requests = lambda s: [self.prepare_request(s, request, opts)] self.spidercls.start_requests = _start_requests @@ -164,7 +168,9 @@ class Command(ScrapyCommand): # determine real callback cb = response.meta['_callback'] if not cb: - if opts.rules and self.first_response == response: + if opts.callback: + cb = opts.callback + elif opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) if not cb: @@ -201,6 +207,10 @@ class Command(ScrapyCommand): req.callback = callback return requests + #update request meta if any extra meta was passed through the --meta/-m opts. + if opts.meta: + request.meta.update(opts.meta) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -208,11 +218,27 @@ class Command(ScrapyCommand): def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) + + self.process_spider_arguments(opts) + self.process_request_meta(opts) + + def process_spider_arguments(self, opts): + try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + def process_request_meta(self, opts): + + if opts.meta: + try: + opts.meta = json.loads(opts.meta) + except ValueError: + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + + def run(self, args, opts): # parse arguments if not len(args) == 1 or not is_url(args[0]): diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 1da09e4da..376d3c84e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -28,6 +28,7 @@ def _import_file(filepath): class Command(ScrapyCommand): requires_project = False + default_settings = {'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options] " @@ -86,3 +87,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bce4e6086..bee52f06a 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options]" diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 594106632..c17aaf442 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn') class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return " [project_dir]" @@ -118,4 +119,4 @@ class Command(ScrapyCommand): _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ join(scrapy.__path__[0], 'templates') return join(_templates_base_dir, 'project') - + diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index a9954edb0..577365c3b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,17 +1,14 @@ from __future__ import print_function -import sys -import platform - -import twisted -import OpenSSL import scrapy from scrapy.commands import ScrapyCommand +from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[-v]" @@ -26,38 +23,11 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - import cssselect - import parsel - import lxml.etree - import w3lib - - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - try: - w3lib_version = w3lib.__version__ - except AttributeError: - w3lib_version = "<1.14.3" - - print("Scrapy : %s" % scrapy.__version__) - print("lxml : %s" % lxml_version) - print("libxml2 : %s" % libxml2_version) - print("cssselect : %s" % cssselect.__version__) - print("parsel : %s" % parsel.__version__) - print("w3lib : %s" % w3lib_version) - print("Twisted : %s" % twisted.version.short()) - print("Python : %s" % sys.version.replace("\n", "- ")) - print("pyOpenSSL : %s" % self._get_openssl_version()) - print("Platform : %s" % platform.platform()) + versions = scrapy_components_versions() + width = max(len(n) for (n, _) in versions) + patt = "%-{}s : %s".format(width) + for name, version in versions: + print(patt % (name, version)) else: print("Scrapy %s" % scrapy.__version__) - def _get_openssl_version(self): - try: - openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ - .decode('ascii', errors='replace') - # pyOpenSSL 0.12 does not expose openssl version - except AttributeError: - openssl = 'Unknown OpenSSL version' - - return '{} ({})'.format(OpenSSL.version.__version__, openssl) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a94a89205..783d4c383 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -64,7 +64,7 @@ if twisted_version >= (14, 0, 0): """ Twisted-recommended context factory for web clients. - Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: "The default is to use a BrowserLikePolicyForHTTPS, so unless you have special requirements you can leave this as-is." @@ -100,6 +100,6 @@ else: def getContext(self, hostname=None, port=None): ctx = ClientContextFactory.getContext(self) # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html ctx.set_options(SSL.OP_ALL) return ctx diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py new file mode 100644 index 000000000..d102f2b73 --- /dev/null +++ b/scrapy/core/downloader/handlers/datauri.py @@ -0,0 +1,23 @@ +from w3lib.url import parse_data_uri + +from scrapy.http import TextResponse +from scrapy.responsetypes import responsetypes +from scrapy.utils.decorators import defers + + +class DataURIDownloadHandler(object): + def __init__(self, settings): + super(DataURIDownloadHandler, self).__init__() + + @defers + def download_request(self, request, spider): + uri = parse_data_uri(request.url) + respcls = responsetypes.from_mimetype(uri.media_type) + + resp_kwargs = {} + if (issubclass(respcls, TextResponse) and + uri.media_type.split('/')[0] == 'text'): + charset = uri.media_type_parameters.get('charset') + resp_kwargs['encoding'] = charset + + return respcls(url=request.url, body=uri.data, **resp_kwargs) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1398140b4..933bc7e8d 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -39,12 +39,13 @@ from twisted.internet.protocol import Protocol, ClientCreator from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): self.__filename = filename - self.body = open(filename, "w") if filename else BytesIO() + self.body = open(filename, "wb") if filename else BytesIO() self.size = 0 def dataReceived(self, data): @@ -97,7 +98,7 @@ class FTPDownloadHandler(object): protocol.close() body = protocol.filename or protocol.body.read() headers = {"local filename": protocol.filename or '', "size": protocol.size} - return respcls(url=request.url, status=200, body=body, headers=headers) + return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers) def _failed(self, result, request): message = result.getErrorMessage() @@ -106,6 +107,6 @@ class FTPDownloadHandler(object): if m: ftpcode = m.group() httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) - return Response(url=request.url, status=httpcode, body=message) + return Response(url=request.url, status=httpcode, body=to_bytes(message)) raise result.type(result.value) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index b96c8c6fe..038db7b47 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,9 +12,13 @@ from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError -from twisted.web.http import PotentialDataLoss +from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool + HTTPConnectionPool, ResponseFailed +try: + from twisted.web.client import URI +except ImportError: + from twisted.web.client import _URI as URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -51,13 +55,15 @@ class HTTP11DownloadHandler(object): warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') self._disconnect_timeout = 1 def download_request(self, request, spider): """Return a deferred for the HTTP download""" agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool, maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize)) + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + fail_on_dataloss=self._fail_on_dataloss) return agent.download_request(request) def close(self): @@ -226,20 +232,49 @@ class TunnelingAgent(Agent): headers, bodyProducer, requestPath) +class ScrapyProxyAgent(Agent): + + def __init__(self, reactor, proxyURI, + connectTimeout=None, bindAddress=None, pool=None): + super(ScrapyProxyAgent, self).__init__(reactor, + connectTimeout=connectTimeout, + bindAddress=bindAddress, + pool=pool) + self._proxyURI = URI.fromBytes(proxyURI) + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + if twisted_version >= (15, 0, 0): + proxyEndpoint = self._getEndpoint(self._proxyURI) + else: + proxyEndpoint = self._getEndpoint(self._proxyURI.scheme, + self._proxyURI.host, + self._proxyURI.port) + key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) + return self._requestWithEndpoint(key, proxyEndpoint, method, + URI.fromBytes(uri), headers, + bodyProducer, uri) + + class ScrapyAgent(object): _Agent = Agent - _ProxyAgent = ProxyAgent + _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0): + maxsize=0, warnsize=0, fail_on_dataloss=True): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress self._pool = pool self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss self._txresponse = None def _get_agent(self, request, timeout): @@ -257,9 +292,8 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, - timeout=timeout, bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'), + connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) @@ -276,8 +310,7 @@ class ScrapyAgent(object): headers.removeHeader(b'Proxy-Authorization') if request.body: bodyproducer = _RequestBodyProducer(request.body) - else: - bodyproducer = None + elif method == b'POST': # Setting Content-Length: 0 even for POST requests is not a # MUST per HTTP RFCs, but it's common behavior, and some # servers require this, otherwise returning HTTP 411 Length required @@ -286,10 +319,13 @@ class ScrapyAgent(object): # "a Content-Length header field is normally sent in a POST # request even when the value is 0 (indicating an empty payload body)." # - # Twisted Agent will not add "Content-Length: 0" by itself - if method == b'POST': - headers.addRawHeader(b'Content-Length', b'0') - + # Twisted < 17 will not add "Content-Length: 0" by itself; + # Twisted >= 17 fixes this; + # Using a producer with an empty-string sends `0` as Content-Length + # for all versions of Twisted. + bodyproducer = _RequestBodyProducer(b'') + else: + bodyproducer = None start_time = time() d = agent.request( method, to_bytes(url, encoding='ascii'), headers, bodyproducer) @@ -326,6 +362,7 @@ class ScrapyAgent(object): maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 + fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: error_msg = ("Cancelling download of %(url)s: expected response " @@ -338,14 +375,16 @@ class ScrapyAgent(object): if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s).", - {'size': expected_size, 'warnsize': warnsize}) + "download warn size (%(warnsize)s) in request %(request)s.", + {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): - txresponse._transport._producer.loseConnection() + # Abort connection immediately. + txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) - txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize)) + txresponse.deliverBody(_ResponseReader( + d, txresponse, request, maxsize, warnsize, fail_on_dataloss)) # save response for timeouts self._txresponse = txresponse @@ -380,25 +419,37 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize): + def __init__(self, finished, txresponse, request, maxsize, warnsize, + fail_on_dataloss): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss + self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): + # This maybe called several times after cancel was called with buffered + # data. + if self._finished.called: + return + self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s).", + "max size (%(maxsize)s) in request %(request)s.", {'bytes': self._bytes_received, - 'maxsize': self._maxsize}) + 'maxsize': self._maxsize, + 'request': self._request}) + # Clear buffer earlier to avoid keeping data in memory for a long + # time. + self._bodybuf.truncate(0) self._finished.cancel() if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: @@ -415,7 +466,22 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss): + return + + if reason.check(PotentialDataLoss): self._finished.callback((self._txresponse, body, ['partial'])) - else: - self._finished.errback(reason) + return + + if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): + if not self._fail_on_dataloss: + self._finished.callback((self._txresponse, body, ['dataloss'])) + return + + elif not self._fail_on_dataloss_warned: + logger.warn("Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode()) + self._fail_on_dataloss_warned = True + + self._finished.errback(reason) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 498e3d60f..df8051182 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,6 +40,14 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) + try: + # XXX: this import would fail on Debian jessie with system installed + # service_identity library, due to lack of cryptography.x509 dependency + # See https://github.com/pyca/service_identity/issues/21 + from service_identity.exceptions import CertificateError + verification_errors = (CertificateError, VerificationError) + except ImportError: + verification_errors = VerificationError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -55,8 +63,9 @@ if twisted_version >= (14, 0, 0): (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError and ValueError exceptions are caught, - so that the connection is not closed, only logging warnings. + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): @@ -65,7 +74,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except VerificationError as e: + except verification_errors as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2b5770138..37fe0a873 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -218,10 +218,8 @@ class ExecutionEngine(object): request=request, spider=spider) def download(self, request, spider): - slot = self.slot - slot.add_request(request) d = self._download(request, spider) - d.addBoth(self._downloaded, slot, request, spider) + d.addBoth(self._downloaded, self.slot, request, spider) return d def _downloaded(self, response, slot, request, spider): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 443a9aa2f..04aee18ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -11,12 +11,14 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader from scrapy.extension import ExtensionManager -from scrapy.settings import Settings +from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object -from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info +from scrapy.utils.log import ( + LogCounterHandler, configure_logging, log_scrapy_info, + get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals logger = logging.getLogger(__name__) @@ -32,11 +34,17 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + d = dict(overridden_settings(self.settings)) + logger.info("Overridden settings: %(settings)r", {'settings': d}) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) - handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL')) + handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) + if get_scrapy_root_handler() is not None: + # scrapy root handler already installed: update it with new settings + install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope self.__remove_handler = lambda: logging.root.removeHandler(handler) @@ -75,7 +83,7 @@ class Crawler(object): yield defer.maybeDeferred(self.engine.start) except Exception: # In Python 2 reraising an exception after yield discards - # the original traceback (see http://bugs.python.org/issue7563), + # the original traceback (see https://bugs.python.org/issue7563), # so sys.exc_info() workaround is used. # This workaround also works in Python 3, but it is not needed, # and it is slower, so in Python 3 we use native `raise`. @@ -129,6 +137,7 @@ class CrawlerRunner(object): self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() + self.bootstrap_failed = False @property def spiders(self): @@ -170,6 +179,7 @@ class CrawlerRunner(object): def _done(result): self.crawlers.discard(crawler) self._active.discard(d) + self.bootstrap_failed |= not getattr(crawler, 'spider', None) return result return d.addBoth(_done) @@ -229,15 +239,18 @@ class CrawlerProcess(CrawlerRunner): The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. + :param install_root_handler: whether to install root logging handler + (default: True) + This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): + def __init__(self, settings=None, install_root_handler=True): super(CrawlerProcess, self).__init__(settings) install_shutdown_handlers(self._signal_shutdown) - configure_logging(self.settings) + configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) def _signal_shutdown(self, signum, _): diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 64d94c489..6748d0265 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -11,7 +11,7 @@ warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as - documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding + documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ def process_response(self, request, response, spider): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..495b103d1 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -75,7 +75,7 @@ class HttpCacheMiddleware(object): return response # RFC2616 requires origin server to set Date header, - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: response.headers['Date'] = formatdate(usegmt=1) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 19d6345e4..203dee42d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured @@ -34,11 +34,11 @@ class HttpCompressionMiddleware(object): return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding and not is_gzipped(response): + if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ - url=response.url) + url=response.url, body=decoded_body) kwargs = dict(cls=respcls, body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..30cae3fee 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) + allowed_status = (301, 302, 303, 307, 308) if 'Location' not in response.headers or response.status not in allowed_status: return response @@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) - if response.status in (301, 307) or request.method == 'HEAD': + if response.status in (301, 307, 308) or request.method == 'HEAD': redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c9c512be8..07e979628 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) @@ -62,7 +63,13 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 - if retries <= self.max_retry_times: + retry_times = self.max_retry_times + + if 'max_retry_times' in request.meta: + retry_times = request.meta['max_retry_times'] + + stats = spider.crawler.stats + if retries <= retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) @@ -70,8 +77,15 @@ class RetryMiddleware(object): retryreq.meta['retry_times'] = retries retryreq.dont_filter = True retryreq.priority = request.priority + self.priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value('retry/count') + stats.inc_value('retry/reason_count/%s' % reason) return retryreq else: + stats.inc_value('retry/max_reached') logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..200245210 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,11 +41,13 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch( - to_native_str(self._useragent), request.url): + if rp is None: + return + if not rp.can_fetch(to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) - raise IgnoreRequest() + self.crawler.stats.inc_value('robotstxt/forbidden') + raise IgnoreRequest("Forbidden by robots.txt") def robot_parser(self, request, spider): url = urlparse_cached(request) @@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object): dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) + self.crawler.stats.inc_value('robotstxt/request_count') if isinstance(self._parsers[netloc], Deferred): d = Deferred() @@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object): return failure def _parse_robots(self, response, netloc): + self.crawler.stats.inc_value('robotstxt/response_count') + self.crawler.stats.inc_value( + 'robotstxt/response_status_count/{}'.format(response.status)) rp = robotparser.RobotFileParser(response.url) body = '' if hasattr(response, 'text'): body = response.text - else: # last effort try + else: # last effort try try: body = response.body.decode('utf-8') except UnicodeDecodeError: @@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object): # but keep the lookup cached (in self._parsers) # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. - pass + self.crawler.stats.inc_value('robotstxt/unicode_error_count') # stdlib's robotparser expects native 'str' ; # with unicode input, non-ASCII encoded bytes decoding fails in Python2 rp.parse(to_native_str(body).splitlines()) @@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): + if failure.type is not IgnoreRequest: + key = 'robotstxt/exception_count/{}'.format(failure.type) + self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None rp_dfd.callback(None) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 9c0ad90a5..ef0aafce0 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,8 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr +from scrapy.utils.python import global_object_name + class DownloaderStats(object): @@ -27,6 +29,6 @@ class DownloaderStats(object): return response def process_exception(self, request, exception, spider): - ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) + ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c4b1b3476..07f43b494 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -36,6 +36,7 @@ class BaseItemExporter(object): self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) + self.indent = options.pop('indent', None) if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -98,21 +99,33 @@ class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) self.file = file + # there is a small difference between the behaviour or JsonItemExporter.indent + # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent + # the addition of newlines everywhere + json_indent = self.indent if self.indent is not None and self.indent > 0 else None + kwargs.setdefault('indent', json_indent) kwargs.setdefault('ensure_ascii', not self.encoding) self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True + def _beautify_newline(self): + if self.indent is not None: + self.file.write(b'\n') + def start_exporting(self): - self.file.write(b"[\n") + self.file.write(b"[") + self._beautify_newline() def finish_exporting(self): - self.file.write(b"\n]") + self._beautify_newline() + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(b',\n') + self.file.write(b',') + self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) self.file.write(to_bytes(data, self.encoding)) @@ -128,35 +141,54 @@ class XmlItemExporter(BaseItemExporter): self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) + def _beautify_newline(self, new_item=False): + if self.indent is not None and (self.indent > 0 or new_item): + self._xg_characters('\n') + + def _beautify_indent(self, depth=1): + if self.indent: + self._xg_characters(' ' * self.indent * depth) + def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) + self._beautify_newline(new_item=True) def export_item(self, item): + self._beautify_indent(depth=1) self.xg.startElement(self.item_element, {}) + self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=''): - self._export_xml_field(name, value) + self._export_xml_field(name, value, depth=2) + self._beautify_indent(depth=1) self.xg.endElement(self.item_element) + self._beautify_newline(new_item=True) def finish_exporting(self): self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value): + def _export_xml_field(self, name, serialized_value, depth): + self._beautify_indent(depth=depth) self.xg.startElement(name, {}) if hasattr(serialized_value, 'items'): + self._beautify_newline() for subname, value in serialized_value.items(): - self._export_xml_field(subname, value) + self._export_xml_field(subname, value, depth=depth+1) + self._beautify_indent(depth=depth) elif is_listlike(serialized_value): + self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value) + self._export_xml_field('value', value, depth=depth+1) + self._beautify_indent(depth=depth) elif isinstance(serialized_value, six.text_type): self._xg_characters(serialized_value) else: self._xg_characters(str(serialized_value)) self.xg.endElement(name) + self._beautify_newline() - # Workaround for http://bugs.python.org/issue17606 + # Workaround for https://bugs.python.org/issue17606 # Before Python 2.7.4 xml.sax.saxutils required bytes; # since 2.7.4 it requires unicode. The bug is likely to be # fixed in 2.7.6, but 2.7.6 will still support unicode, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 85d328528..5f133fbde 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -172,6 +172,9 @@ class FeedExporter(object): self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.indent = None + if settings.get('FEED_EXPORT_INDENT') is not None: + self.indent = settings.getint('FEED_EXPORT_INDENT') uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,7 +191,7 @@ class FeedExporter(object): storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding) + encoding=self.export_encoding, indent=self.indent) if self.store_empty: exporter.start_exporting() self._exporting = True diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 247cac64e..1b5e05b1b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import gzip +import logging from six.moves import cPickle as pickle from importlib import import_module from time import time @@ -12,7 +13,10 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes, to_unicode, garbage_collect + + +logger = logging.getLogger(__name__) class DummyPolicy(object): @@ -66,8 +70,8 @@ class RFC2616Policy(object): return True def should_cache_response(self, response, request): - # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 - # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" @@ -159,7 +163,7 @@ class RFC2616Policy(object): def _compute_freshness_lifetime(self, response, request, now): # Reference nsHttpResponseHead::ComputeFreshnessLifetime - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) maxage = self._get_max_age(cc) if maxage is not None: @@ -190,7 +194,7 @@ class RFC2616Policy(object): def _compute_current_age(self, response, request, now): # Reference nsHttpResponseHead::ComputeCurrentAge - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#366 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server @@ -220,6 +224,8 @@ class DbmCacheStorage(object): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') + logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): self.db.close() @@ -272,7 +278,8 @@ class FilesystemCacheStorage(object): self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - pass + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + extra={'spider': spider}) def close_spider(self, spider): pass @@ -348,11 +355,14 @@ class LeveldbCacheStorage(object): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) + logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): # Do compactation each time to save space and also recreate files to # avoid them being removed in storages with timestamp-based autoremoval. self.db.CompactRange() del self.db + garbage_collect() def retrieve_response(self, spider, request): data = self._read_data(spider, request) @@ -404,7 +414,7 @@ class LeveldbCacheStorage(object): def parse_cachecontrol(header): """Parse Cache-Control header - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None, ... b'max-age': b'3600'} diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 322213cf0..c0570567e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -35,7 +35,6 @@ class MemoryUsage(object): self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 - self.report = crawler.settings.getbool('MEMUSAGE_REPORT') self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index d9add1d97..e78afa1fc 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -55,9 +55,9 @@ class TelnetConsole(protocol.ServerFactory): def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() - logger.debug("Telnet console listening on %(host)s:%(port)d", - {'host': h.host, 'port': h.port}, - extra={'crawler': self.crawler}) + logger.info("Telnet console listening on %(host)s:%(port)d", + {'host': h.host, 'port': h.port}, + extra={'crawler': self.crawler}) def stop_listening(self): self.port.stopListening() @@ -82,7 +82,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " \ - "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a1e95102e..4e8056750 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -58,8 +58,8 @@ class CookieJar(object): def clear_session_cookies(self, *args, **kwargs): return self.jar.clear_session_cookies(*args, **kwargs) - def clear(self): - return self.jar.clear() + def clear(self, domain=None, path=None, name=None): + return self.jar.clear(domain, path, name) def __iter__(self): return iter(self.jar) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1435d91de..13a92ffa0 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,6 +27,10 @@ class Request(object_ref): assert isinstance(priority, int), "Request priority not an integer: %r" % priority self.priority = priority + if callback is not None and not callable(callback): + raise TypeError('callback must be a callable, got %s' % type(callback).__name__) + if errback is not None and not callable(errback): + raise TypeError('errback must be a callable, got %s' % type(errback).__name__) assert callback or not errback, "Cannot use errback without a callback" self.callback = callback self.errback = errback diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 905d8412f..95b38e990 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -135,7 +135,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend(formdata.items()) + values.extend((k, v) for k, v in formdata.items() if v is not None) return values @@ -170,9 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::button[not(@type)]', + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 434d87eab..1974259b5 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -120,6 +120,8 @@ class Response(object_ref): """ if isinstance(url, Link): url = url.url + elif url is None: + raise ValueError("url can't be None") url = self.urljoin(url) return Request(url, callback, method=method, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6415e191a..74a042f2c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -135,7 +135,7 @@ class TextResponse(Response): * an attribute Selector (not SelectorList) - e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]``. - * a Selector for ```` element, e.g. + * a Selector for ```` or ```` element, e.g. ``response.css('a.my_link')[0]``. See :ref:`response-follow-example` for usage examples. @@ -165,10 +165,11 @@ def _url_from_selector(sel): return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): raise ValueError("Unsupported selector: %s" % sel) - if sel.root.tag != 'a': - raise ValueError("Only elements are supported; got <%s>" % + if sel.root.tag not in ('a', 'link'): + raise ValueError("Only and elements are supported; got <%s>" % sel.root.tag) href = sel.root.get('href') if href is None: - raise ValueError(" element has no href attribute: %s" % sel) + raise ValueError("<%s> element has no href attribute: %s" % + (sel.root.tag, sel)) return strip_html5_whitespace(href) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index e5d21e174..c3c79cf25 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', + 'm4a', 'm4v', 'flv', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', @@ -101,7 +101,7 @@ class FilteringLinkExtractor(object): links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: - link.url = canonicalize_url(urlparse(link.url)) + link.url = canonicalize_url(link.url) links = self.link_extractor._process_links(links) return links diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index c284f1905..a7092f9b8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,6 +6,7 @@ from six.moves.urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace +from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -29,12 +30,17 @@ def _nons(tag): class LxmlParserLinkExtractor(object): def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True): + strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _iter_links(self, document): for el in document.iter(etree.Element): @@ -82,21 +88,27 @@ class LxmlParserLinkExtractor(object): def _deduplicate_if_needed(self, links): if self.unique: - return unique_list(links, key=lambda link: link.url) + return unique_list(links, key=self.link_key) return links class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process=process_value, strip=strip) + lx = LxmlParserLinkExtractor( + tag=tag_func, + attr=attr_func, + unique=unique, + process=process_value, + strip=strip, + canonicalized=canonicalize + ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 11ff7a261..f4ca4262a 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -6,7 +6,7 @@ from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser -from w3lib.url import safe_url_string +from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace from scrapy.link import Link @@ -20,7 +20,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning class BaseSgmlLinkExtractor(SGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None, - strip=True): + strip=True, canonicalized=False): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -33,6 +33,11 @@ class BaseSgmlLinkExtractor(SGMLParser): self.current_link = None self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _extract_links(self, response_text, response_url, response_encoding, base_url=None): """ Do the real extraction work """ @@ -61,8 +66,7 @@ class BaseSgmlLinkExtractor(SGMLParser): The subclass should override it if necessary """ - links = unique_list(links, key=lambda link: link.url) if self.unique else links - return links + return unique_list(links, key=self.link_key) if self.unique else links def extract_links(self, response): # wrapper needed to allow to work directly with text @@ -107,10 +111,9 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): - warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -124,7 +127,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index e7bf7942e..075a6d862 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -43,6 +43,8 @@ class LogFormatter(object): 'request_flags' : request_flags, 'referer': referer_str(request), 'response_flags': response_flags, + # backward compatibility with Scrapy logformatter below 1.4 version + 'flags': response_flags } } diff --git a/scrapy/mail.py b/scrapy/mail.py index 0bb395521..5b944e1c4 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -5,7 +5,10 @@ See documentation in docs/topics/email.rst """ import logging -from six.moves import cStringIO as StringIO +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO import six from email.utils import COMMASPACE, formatdate @@ -21,19 +24,26 @@ else: from twisted.internet import defer, reactor, ssl -from .utils.misc import arg_to_iter +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +def _to_bytes_or_none(text): + if text is None: + return None + return to_bytes(text) + + class MailSender(object): def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = smtpuser - self.smtppass = smtppass + self.smtpuser = _to_bytes_or_none(smtpuser) + self.smtppass = _to_bytes_or_none(smtppass) self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom @@ -88,7 +98,7 @@ class MailSender(object): 'mailattachs': len(attachs)}) return - dfd = self._sendmail(rcpts, msg.as_string()) + dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) dfd.addCallbacks(self._sent_ok, self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], errbackArgs=[to, cc, subject, len(attachs)]) @@ -112,7 +122,7 @@ class MailSender(object): def _sendmail(self, to_addrs, msg): # Import twisted.mail here because it is not available in python3 from twisted.mail.smtp import ESMTPSenderFactory - msg = StringIO(msg) + msg = BytesIO(msg) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -40,7 +40,6 @@ class FileException(Exception): class FSFilesStore(object): - def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -58,7 +57,7 @@ class FSFilesStore(object): absolute_path = self._get_filesystem_path(path) try: last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! + except os.error: return {} with open(absolute_path, 'rb') as f: @@ -79,9 +78,12 @@ class FSFilesStore(object): class S3FilesStore(object): - AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_ENDPOINT_URL = None + AWS_REGION_NAME = None + AWS_USE_SSL = None + AWS_VERIFY = None POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in # FilesPipeline.from_settings. @@ -95,8 +97,14 @@ class S3FilesStore(object): import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY) + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection @@ -120,7 +128,7 @@ class S3FilesStore(object): def _get_boto_bucket(self): # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 + # https://bugs.python.org/issue5103 c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) return c.get_bucket(self.bucket, validate=False) @@ -181,7 +189,7 @@ class S3FilesStore(object): 'X-Amz-Grant-Read': 'GrantRead', 'X-Amz-Grant-Read-ACP': 'GrantReadACP', 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', - }) + }) extra = {} for key, value in six.iteritems(headers): try: @@ -194,6 +202,52 @@ class S3FilesStore(object): return extra +class GCSFilesStore(object): + + GCS_PROJECT_ID = None + + CACHE_CONTROL = 'max-age=172800' + + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None + + def __init__(self, uri): + from google.cloud import storage + client = storage.Client(project=self.GCS_PROJECT_ID) + bucket, prefix = uri[5:].split('/', 1) + self.bucket = client.bucket(bucket) + self.prefix = prefix + + def stat_file(self, path, info): + def _onsuccess(blob): + if blob: + checksum = blob.md5_hash + last_modified = time.mktime(blob.updated.timetuple()) + return {'checksum': checksum, 'last_modified': last_modified} + else: + return {} + + return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + + def _get_content_type(self, headers): + if headers and 'Content-Type' in headers: + return headers['Content-Type'] + else: + return 'application/octet-stream' + + def persist_file(self, path, buf, info, meta=None, headers=None): + blob = self.bucket.blob(self.prefix + path) + blob.cache_control = self.CACHE_CONTROL + blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + return threads.deferToThread( + blob.upload_from_string, + data=buf.getvalue(), + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY + ) + + class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading @@ -219,6 +273,7 @@ class FilesPipeline(MediaPipeline): '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, + 'gs': GCSFilesStore, } DEFAULT_FILES_URLS_FIELD = 'file_urls' DEFAULT_FILES_RESULT_FIELD = 'files' @@ -226,7 +281,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -249,15 +304,23 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func) + super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None + store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) @@ -423,4 +486,5 @@ class FilesPipeline(MediaPipeline): # deprecated def file_key(self, url): return self.file_path(url) + file_key._base = True diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5796bfb80..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -91,6 +91,10 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) @@ -132,6 +136,11 @@ class ImagesPipeline(FilesPipeline): background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') + elif image.mode == 'P': + image = image.convert("RGBA") + background = Image.new('RGBA', image.size, (255, 255, 255)) + background.paste(image, image) + image = background.convert('RGB') elif image.mode != 'RGB': image = image.convert('RGB') diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 57f70499e..404bbf5bf 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,10 +1,13 @@ from __future__ import print_function +import functools import logging from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.settings import Settings +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -24,9 +27,23 @@ class MediaPipeline(object): self.downloaded = {} self.waiting = defaultdict(list) - def __init__(self, download_func=None): + def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: + settings = Settings(settings) + resolve = functools.partial(self._key_for_pipe, + base_class_name="MediaPipeline", + settings=settings) + self.allow_redirects = settings.getbool( + resolve('MEDIA_ALLOW_REDIRECTS'), False + ) + self._handle_statuses(self.allow_redirects) + + def _handle_statuses(self, allow_redirects): + self.handle_httpstatus_list = None + if allow_redirects: + self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -93,6 +110,12 @@ class MediaPipeline(object): ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _modify_media_request(self, request): + if self.handle_httpstatus_list: + request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list + else: + request.meta['handle_httpstatus_all'] = True + def _check_media_to_download(self, result, request, info): if result is not None: return result @@ -103,7 +126,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request.meta['handle_httpstatus_all'] = True + self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 4f4f0b04f..0aaced7e4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver): # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) d = super(CachingThreadedResolver, self).getHostByName(name, timeout) - d.addCallback(self._cache_result, name) + if dnscache.limit: + d.addCallback(self._cache_result, name) return d def _cache_result(self, result, name): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -13,7 +13,6 @@ Scrapy developers, if you add a setting here remember to: """ -import os import sys from importlib import import_module from os.path import join, abspath, dirname @@ -67,6 +66,7 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', @@ -79,6 +79,8 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' @@ -111,13 +113,9 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter' -try: - EDITOR = os.environ['EDITOR'] -except KeyError: - if sys.platform == 'win32': - EDITOR = '%s -m idlelib.idle' - else: - EDITOR = 'vi' +EDITOR = 'vi' +if sys.platform == 'win32': + EDITOR = '%s -m idlelib.idle' EXTENSIONS = {} @@ -158,8 +156,10 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } +FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' @@ -215,7 +216,6 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] -MEMUSAGE_REPORT = False MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True @@ -232,10 +232,11 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False @@ -246,6 +247,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' +SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} @@ -270,7 +272,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/shell.py b/scrapy/shell.py index 6f94635a1..80b625633 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -164,7 +164,7 @@ class Shell(object): def inspect_response(response, spider): """Open a shell to inspect the given response""" - Shell(spider.crawler).start(response=response) + Shell(spider.crawler).start(response=response, spider=spider) def _request_deferred(request): diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index fd79905e9..296d27ed8 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -55,7 +55,7 @@ class SignalManager(object): The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html + .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d4f0f663f..7478faa78 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from collections import defaultdict import traceback import warnings @@ -18,11 +19,26 @@ class SpiderLoader(object): """ def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') + self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') self._spiders = {} + self._found = defaultdict(list) self._load_all_spiders() + def _check_name_duplicates(self): + dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( + module=mod, cls=cls, name=name) + for (mod, cls) in locations) + for name, locations in self._found.items() + if len(locations)>1] + if dupes: + msg = ("There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format( + "\n\n".join(dupes))) + warnings.warn(msg, UserWarning) + def _load_spiders(self, module): for spcls in iter_spider_classes(module): + self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls def _load_all_spiders(self): @@ -31,10 +47,14 @@ class SpiderLoader(object): for module in walk_modules(name): self._load_spiders(module) except ImportError as e: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "Check SPIDER_MODULES setting".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + if self.warn_only: + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc())) + warnings.warn(msg, RuntimeWarning) + else: + raise + self._check_name_duplicates() @classmethod def from_settings(cls, settings): diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index ea1c9270f..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst import re import logging +import warnings from scrapy import signals from scrapy.http import Request @@ -52,9 +53,18 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + url_pattern = re.compile("^https?://.*$") + for domain in allowed_domains: + if url_pattern.match(domain): + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) def spider_opened(self, spider): self.host_regex = self.get_host_regex(spider) self.domains_seen = set() + + +class URLWarning(Warning): + pass diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6a8c46543..1ddfb37f4 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,22 +2,359 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse +import warnings -from scrapy.http import Request +from w3lib.url import safe_url_string + +from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured +from scrapy import signals +from scrapy.utils.python import to_native_str +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object +from scrapy.utils.url import strip_url + + +LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) + +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_STRICT_ORIGIN = "strict-origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + +class ReferrerPolicy(object): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + + def referrer(self, response_url, request_url): + raise NotImplementedError() + + def stripped_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(url) + + def origin_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(url) + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if not url: + return None + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) + + def origin(self, url): + """Return serialized origin (scheme, host, path) for a request or response URL.""" + return self.strip_url(url, origin_only=True) + + def potentially_trustworthy(self, url): + # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy + parsed_url = urlparse(url) + if parsed_url.scheme in ('data',): + return False + return self.tls_protected(url) + + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') + + +class NoReferrerPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer + + The simplest policy is "no-referrer", which specifies that no referrer information + is to be sent along with requests made from a particular request client to any origin. + The header will be omitted entirely. + """ + name = POLICY_NO_REFERRER + + def referrer(self, response_url, request_url): + return None + + +class NoReferrerWhenDowngradePolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + The "no-referrer-when-downgrade" policy sends a full URL along with requests + from a TLS-protected environment settings object to a potentially trustworthy URL, + and requests from clients which are not TLS-protected to any origin. + + Requests from TLS-protected clients to non-potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + + This is a user agent's default behavior, if no policy is otherwise specified. + """ + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + + def referrer(self, response_url, request_url): + if not self.tls_protected(response_url) or self.tls_protected(request_url): + return self.stripped_referrer(response_url) + + +class SameOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin + + The "same-origin" policy specifies that a full URL, stripped for use as a referrer, + is sent as referrer information when making same-origin requests from a particular request client. + + Cross-origin requests, on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_SAME_ORIGIN + + def referrer(self, response_url, request_url): + if self.origin(response_url) == self.origin(request_url): + return self.stripped_referrer(response_url) + + +class OriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin + + The "origin" policy specifies that only the ASCII serialization + of the origin of the request client is sent as referrer information + when making both same-origin requests and cross-origin requests + from a particular request client. + """ + name = POLICY_ORIGIN + + def referrer(self, response_url, request_url): + return self.origin_referrer(response_url) + + +class StrictOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin + + The "strict-origin" policy sends the ASCII serialization + of the origin of the request client when making requests: + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected request clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN + + def referrer(self, response_url, request_url): + if ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) + + +class OriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin + + The "origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + is sent as referrer information when making cross-origin requests + from a particular request client. + """ + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + else: + return origin + + +class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin + + The "strict-origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + when making cross-origin requests: + + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + elif ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) + + +class UnsafeUrlPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer, + is sent along with both cross-origin requests + and same-origin requests made from a particular request client. + + Note: The policy's name doesn't lie; it is unsafe. + This policy will leak origins and paths from TLS-protected resources + to insecure origins. + Carefully consider the impact of setting such a policy for potentially sensitive documents. + """ + name = POLICY_UNSAFE_URL + + def referrer(self, response_url, request_url): + return self.stripped_referrer(response_url) + + +class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ + NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT + + +_policy_classes = {p.name: p for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, +)} + +# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string +_policy_classes[''] = NoReferrerWhenDowngradePolicy + + +def _load_policy_class(policy, warning_only=False): + """ + Expect a string for the path to the policy class, + otherwise try to interpret the string as a standard value + from https://www.w3.org/TR/referrer-policy/#referrer-policies + """ + try: + return load_object(policy) + except ValueError: + try: + return _policy_classes[policy.lower()] + except KeyError: + msg = "Could not load referrer policy %r" % policy + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None + class RefererMiddleware(object): + def __init__(self, settings=None): + self.default_policy = DefaultReferrerPolicy + if settings is not None: + self.default_policy = _load_policy_class( + settings.get('REFERRER_POLICY')) + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + mw = cls(crawler.settings) + + # Note: this hook is a bit of a hack to intercept redirections + crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + + return mw + + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + + - if a valid policy is set in Request meta, it is used. + - if the policy is set in meta but is wrong (e.g. a typo error), + the policy from settings is used + - if the policy is not set in Request meta, + but there is a Referrer-policy header in the parent response, + it is used if valid + - otherwise, the policy from settings is used. + """ + policy_name = request.meta.get('referrer_policy') + if policy_name is None: + if isinstance(resp_or_url, Response): + policy_header = resp_or_url.headers.get('Referrer-Policy') + if policy_header is not None: + policy_name = to_native_str(policy_header.decode('latin1')) + if policy_name is None: + return self.default_policy() + + cls = _load_policy_class(policy_name, warning_only=True) + return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) + referrer = self.policy(response, r).referrer(response.url, r.url) + if referrer is not None: + r.headers.setdefault('Referer', referrer) return r return (_set_referer(r) for r in result or ()) + def request_scheduled(self, request, spider): + # check redirected request to patch "Referer" header if necessary + redirected_urls = request.meta.get('redirect_urls', []) + if redirected_urls: + request_referrer = request.headers.get('Referer') + # we don't patch the referrer value if there is none + if request_referrer is not None: + # the request's referrer header value acts as a surrogate + # for the parent response URL + # + # Note: if the 3xx response contained a Referrer-Policy header, + # the information is not available using this hook + parent_url = safe_url_string(request_referrer) + policy_referrer = self.policy(parent_url, request).referrer( + parent_url, request.url) + if policy_referrer != request_referrer: + if policy_referrer is None: + request.headers.pop('Referer') + else: + request.headers['Referer'] = policy_referrer diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30cb7590a..e9c131e3b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError + raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod def update_settings(cls, settings): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 9e45637c3..0ee8ba5e7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,8 @@ import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip, gzip_magic_number + logger = logging.getLogger(__name__) @@ -47,7 +48,7 @@ class SitemapSpider(Spider): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s): + for loc in iterloc(s, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) @@ -59,12 +60,19 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif is_gzipped(response): + elif gzip_magic_number(response): return gunzip(response.body) - elif response.url.endswith('.xml'): + # actual gzipped sitemap files are decompressed above ; + # if we are here (response body is not gzipped) + # and have a response for .xml.gz, + # it usually means that it was already gunzipped + # by HttpCompression middleware, + # the HTTP response being sent with "Content-Encoding: gzip" + # without actually being a .xml.gz file in the first place, + # merely XML gzip-compressed on the fly, + # in other word, here, we have plain XML + elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body - elif response.url.endswith('.xml.gz'): - return gunzip(response.body) def regex(x): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 21520f454..d2074a457 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python>=3.5 raises AttributeError here while - # Python<=3.4 raises pickle.PicklingError - except (pickle.PicklingError, AttributeError) as e: + # Python <= 3.4 raises pickle.PicklingError here while + # 3.5 <= Python < 3.6 raises AttributeError and + # Python >= 3.6 raises TypeError + except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 2c746138f..7d766f4fc 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/items.html +# https://doc.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 42318fec2..c5b542bd6 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals @@ -20,14 +20,14 @@ class ${ProjectName}SpiderMiddleware(object): crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s - def process_spider_input(response, spider): + def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None - def process_spider_output(response, result, spider): + def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. @@ -35,7 +35,7 @@ class ${ProjectName}SpiderMiddleware(object): for i in result: yield i - def process_spider_exception(response, exception, spider): + def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. @@ -43,7 +43,7 @@ class ${ProjectName}SpiderMiddleware(object): # or Item objects. pass - def process_start_requests(start_requests, spider): + def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. @@ -54,3 +54,50 @@ class ${ProjectName}SpiderMiddleware(object): def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) + + +class ${ProjectName}DownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4e9b32e9e..e58dab089 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 486df6b71..a0557473e 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# http://doc.scrapy.org/en/latest/topics/settings.html -# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html -# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, +# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html +# See https://doc.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See http://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg index d7f02e0a2..1daeaa541 100644 --- a/scrapy/templates/project/scrapy.cfg +++ b/scrapy/templates/project/scrapy.cfg @@ -1,7 +1,7 @@ # Automatically created by: scrapy startproject # # For more information about the [deploy] section see: -# https://scrapyd.readthedocs.org/en/latest/deploy.html +# https://scrapyd.readthedocs.io/en/latest/deploy.html [settings] default = ${project_name}.settings diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 99e5d43b2..1cfe9cc9d 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -3,8 +3,8 @@ import scrapy class $classname(scrapy.Spider): - name = "$name" - allowed_domains = ["$domain"] + name = '$name' + allowed_domains = ['$domain'] start_urls = ['http://$domain/'] def parse(self, response): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 1888d9599..2e9981556 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -31,6 +31,15 @@ def _embed_bpython_shell(namespace={}, banner=''): bpython.embed(locals_=namespace, banner=banner) return wrapper +def _embed_ptpython_shell(namespace={}, banner=''): + """Start a ptpython shell""" + import ptpython.repl + @wraps(_embed_ptpython_shell) + def wrapper(namespace=namespace, banner=''): + print(banner) + ptpython.repl.embed(locals=namespace) + return wrapper + def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code @@ -47,9 +56,10 @@ def _embed_standard_shell(namespace={}, banner=''): return wrapper DEFAULT_PYTHON_SHELLS = OrderedDict([ + ('ptpython', _embed_ptpython_shell), ('ipython', _embed_ipython_shell), ('bpython', _embed_bpython_shell), - ( 'python', _embed_standard_shell), + ('python', _embed_standard_shell), ]) def get_shell_embed_func(shells=None, known_shells=None): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e516185bd..eb373c501 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,7 +8,7 @@ This module must not depend on any module outside the Standard Library. import copy import six import warnings -from collections import OrderedDict +from collections import OrderedDict, Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -224,7 +224,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - seq = seq.items() if isinstance(seq, dict) else seq + seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super(CaselessDict, self).update(iseq) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bb4c74a6e..bcf209511 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() @@ -57,7 +57,7 @@ def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. - Taken from: http://jcalderone.livejournal.com/24285.html + Taken from: https://jcalderone.livejournal.com/24285.html """ coop = task.Cooperator() work = (callable(elem, *args, **named) for elem in iterable) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 7ab39c97e..f76161a68 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -71,8 +71,8 @@ def create_deprecated_class(name, new_class, clsdict=None, warnings.warn(msg, warn_category, stacklevel=2) super(DeprecatedClass, cls).__init__(name, bases, clsdict_) - # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass - # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks + # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass + # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) @@ -159,10 +159,10 @@ def update_classpath(path): def method_is_overridden(subclass, base_class, method_name): - """ - Return True if a method named ``method_name`` of a ``base_class`` - is overridden in a ``subclass``. - + """ + Return True if a method named ``method_name`` of a ``base_class`` + is overridden in a ``subclass``. + >>> class Base(object): ... def foo(self): ... pass diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 73c2eb73b..ec3949651 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -30,25 +30,25 @@ def gunzip(data): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output = b'' + output_list = [] chunk = b'.' while chunk: try: chunk = read1(f, 8196) - output += chunk + output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output is '' and f.extrabuf + # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output or getattr(f, 'extrabuf', None): + if output_list or getattr(f, 'extrabuf', None): try: - output += f.extrabuf[-f.extrasize:] + output_list.append(f.extrabuf[-f.extrasize:]) finally: break else: raise - return output + return b''.join(output_list) _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search @@ -59,3 +59,7 @@ def is_gzipped(response): cenc = response.headers.get('Content-Encoding', b'').lower() return (_is_gzipped(ctype) or (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + + +def gzip_magic_number(response): + return response.body[:3] == b'\x1f\x8b\x08' diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index 8b659a22a..7cc8d1884 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -11,7 +11,7 @@ def decode_chunked_transfer(chunked_body): decoded body. For more info see: - http://en.wikipedia.org/wiki/Chunked_transfer_encoding + https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ body, h, t = '', '', chunked_body diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index f33ce7017..828880709 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -9,8 +9,10 @@ from twisted.python.failure import Failure from twisted.python import log as twisted_log import scrapy -from scrapy.settings import overridden_settings, Settings +from scrapy.settings import Settings from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.versions import scrapy_components_versions + logger = logging.getLogger(__name__) @@ -96,9 +98,25 @@ def configure_logging(settings=None, install_root_handler=True): sys.stdout = StreamLogger(logging.getLogger('stdout')) if install_root_handler: - logging.root.setLevel(logging.NOTSET) - handler = _get_handler(settings) - logging.root.addHandler(handler) + install_scrapy_root_handler(settings) + + +def install_scrapy_root_handler(settings): + global _scrapy_root_handler + + if (_scrapy_root_handler is not None + and _scrapy_root_handler in logging.root.handlers): + logging.root.removeHandler(_scrapy_root_handler) + logging.root.setLevel(logging.NOTSET) + _scrapy_root_handler = _get_handler(settings) + logging.root.addHandler(_scrapy_root_handler) + + +def get_scrapy_root_handler(): + return _scrapy_root_handler + + +_scrapy_root_handler = None def _get_handler(settings): @@ -126,16 +144,17 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - - d = dict(overridden_settings(settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) + logger.info("Versions: %(versions)s", + {'versions': ", ".join("%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy")}) class StreamLogger(object): """Fake file-like stream object that redirects writes to a logger instance Taken from: - http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ + https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ def __init__(self, logger, log_level=logging.INFO): self.logger = logger diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 30c9e5058..35f855007 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index df4eee5ec..f87d5a803 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,17 +1,18 @@ from __future__ import absolute_import +import signal from twisted.internet import reactor -import signal signal_names = {} for signame in dir(signal): - if signame.startswith("SIG"): + if signame.startswith('SIG') and not signame.startswith('SIG_'): signum = getattr(signal, signame) if isinstance(signum, int): signal_names[signum] = signame + def install_shutdown_handlers(function, override_sigint=True): """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the @@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True): override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows - if hasattr(signal, "SIGBREAK"): + if hasattr(signal, 'SIGBREAK'): signal.signal(signal.SIGBREAK, function) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 42fbbda7f..732ca13a0 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import gc import os import re import inspect @@ -8,6 +9,7 @@ import weakref import errno import six from functools import partial, wraps +import sys from scrapy.utils.decorators import deprecated @@ -195,10 +197,30 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) +def _getargspec_py23(func): + """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, + defaults) + + Identical to inspect.getargspec() in python2, but uses + inspect.getfullargspec() for python3 behind the scenes to avoid + DeprecationWarning. + + >>> def f(a, b=2, *ar, **kw): + ... pass + + >>> _getargspec_py23(f) + ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) + """ + if six.PY2: + return inspect.getargspec(func) + + return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) + + def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = inspect.getargspec(func) + func_args, _, _, _ = _getargspec_py23(func) elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): @@ -245,9 +267,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = inspect.getargspec(func) + spec = _getargspec_py23(func) elif hasattr(func, '__call__'): - spec = inspect.getargspec(func.__call__) + spec = _getargspec_py23(func.__call__) else: raise TypeError('%s is not callable' % type(func)) @@ -344,3 +366,24 @@ def without_none_values(iterable): return {k: v for k, v in six.iteritems(iterable) if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) + + +def global_object_name(obj): + """ + Return full name of a global object. + + >>> from scrapy import Request + >>> global_object_name(Request) + 'scrapy.http.request.Request' + """ + return "%s.%s" % (obj.__module__, obj.__name__) + + +if hasattr(sys, "pypy_version_info"): + def garbage_collect(): + # Collecting weakreferences can take two collections on PyPy. + gc.collect() + gc.collect() +else: + def garbage_collect(): + gc.collect() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index deb5741be..bf276b5ca 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -43,7 +43,8 @@ def get_meta_refresh(response): def response_status_message(status): """Return status code plus status text descriptive message """ - return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status"))) + message = http.RESPONSES.get(int(status), "Unknown Status") + return '%s %s' % (status, to_native_str(message)) def response_httprepr(response): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d2ef68912..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,6 +20,12 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") + +def assert_gcs_environ(): + if 'GCS_PROJECT_ID' not in os.environ: + raise SkipTest("GCS_PROJECT_ID not found") + + def skip_if_no_boto(): try: is_botocore() @@ -45,6 +51,16 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content +def get_gcs_content_and_delete(bucket, path): + from google.cloud import storage + client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted + bucket.delete_blob(path) + return content, acl, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..657c53815 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse) +from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code @@ -47,7 +47,7 @@ def parse_url(url, encoding=None): def escape_ajax(url): """ Return the crawleable url according to: - http://code.google.com/web/ajaxcrawling/docs/getting-started.html + https://developers.google.com/webmasters/ajax-crawling/docs/getting-started >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' @@ -103,3 +103,34 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): + + """Strip URL string from some of its components: + + - `strip_credentials` removes "user:password@" + - `strip_default_port` removes ":80" (resp. ":443", ":21") + from http:// (resp. https://, ftp://) URLs + - `origin_only` replaces path component with "/", also dropping + query and fragment components ; it also strips credentials + - `strip_fragment` drops any #fragment component + """ + + parsed_url = urlparse(url) + netloc = parsed_url.netloc + if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): + netloc = netloc.split('@')[-1] + if strip_default_port and parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), + ('https', 443), + ('ftp', 21)): + netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + return urlunparse(( + parsed_url.scheme, + netloc, + '/' if origin_only else parsed_url.path, + '' if origin_only else parsed_url.params, + '' if origin_only else parsed_url.query, + '' if strip_fragment else parsed_url.fragment + )) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py new file mode 100644 index 000000000..58c7aef85 --- /dev/null +++ b/scrapy/utils/versions.py @@ -0,0 +1,50 @@ +import platform +import sys + +import cssselect +import lxml.etree +import parsel +import twisted +import w3lib + +import scrapy + + +def scrapy_components_versions(): + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + try: + import cryptography + cryptography_version = cryptography.__version__ + except ImportError: + cryptography_version = "unknown" + + return [ + ("Scrapy", scrapy.__version__), + ("lxml", lxml_version), + ("libxml2", libxml2_version), + ("cssselect", cssselect.__version__), + ("parsel", parsel.__version__), + ("w3lib", w3lib_version), + ("Twisted", twisted.version.short()), + ("Python", sys.version.replace("\n", "- ")), + ("pyOpenSSL", _get_openssl_version()), + ("cryptography", cryptography_version), + ("Platform", platform.platform()), + ] + + +def _get_openssl_version(): + try: + import OpenSSL + openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ + .decode('ascii', errors='replace') + # pyOpenSSL 0.12 does not expose openssl version + except AttributeError: + openssl = 'Unknown OpenSSL version' + + return '{} ({})'.format(OpenSSL.version.__version__, openssl) diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 2f0fe3500..3766f38fc 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see http://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index c0f945b66..7425c0930 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -16,7 +16,7 @@ Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [http://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -58,7 +58,7 @@ As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to `select`, `sel` (is shortness if required), or `xpath` after `lxml's -`_ ``xpath`` method. +`_ ``xpath`` method. Bonus (ItemBuilder) =================== @@ -71,5 +71,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (http://doc.scrapy.org/topics/selectors.html) - 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html) + 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) + 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) diff --git a/sep/sep-013.rst b/sep/sep-013.rst index 4c11a0762..5b18b7501 100644 --- a/sep/sep-013.rst +++ b/sep/sep-013.rst @@ -44,7 +44,7 @@ Overview of changes proposed Most of the inconsistencies come from the fact that middlewares don't follow the typical -[http://twistedmatrix.com/projects/core/documentation/howto/defer.html +[https://twistedmatrix.com/projects/core/documentation/howto/defer.html deferred] callback/errback chaining logic. Twisted logic is fine and quite intuitive, and also fits middlewares very well. Due to some bad design choices the integration between middleware calls and deferred is far from optional. So diff --git a/sep/sep-017.rst b/sep/sep-017.rst index 7707a1622..86005e3c9 100644 --- a/sep/sep-017.rst +++ b/sep/sep-017.rst @@ -13,7 +13,7 @@ SEP-017: Spider Contracts The motivation for Spider Contracts is to build a lightweight mechanism for testing your spiders, and be able to run the tests quickly without having to wait for all the spider to run. It's partially based on the -[http://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach +[https://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach (hence its name) where you define certain conditions that spider callbacks must met, and you give example testing pages. diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 49d068479..52d78097b 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -29,7 +29,7 @@ the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader -is the `definition list `_:: +is the `definition list `_::
diff --git a/setup.py b/setup.py index 086ab8142..c37919cda 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,35 @@ from os.path import dirname, join -from setuptools import setup, find_packages +from pkg_resources import parse_version +from setuptools import setup, find_packages, __version__ as setuptools_version with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() +def has_environment_marker_platform_impl_support(): + """Code extracted from 'pytest/setup.py' + https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 + + The first known release to support environment marker with range operators + it is 18.5, see: + https://setuptools.readthedocs.io/en/latest/history.html#id235 + """ + return parse_version(setuptools_version) >= parse_version('18.5') + + +extras_require = {} + +if has_environment_marker_platform_impl_support(): + extras_require[':platform_python_implementation == "PyPy"'] = [ + 'PyPyDispatcher>=2.1.0', + ] + + setup( name='Scrapy', version=version, - url='http://scrapy.org', + url='https://scrapy.org', description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst').read(), author='Scrapy developers', @@ -33,14 +53,16 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', install_requires=[ 'Twisted>=13.1.0', 'w3lib>=1.17.0', @@ -49,8 +71,9 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.1', + 'parsel>=1.4', 'PyDispatcher>=2.0.5', 'service_identity', ], + extras_require=extras_require, ) diff --git a/tests/__init__.py b/tests/__init__.py index c2e4fd2bf..55b1ecde8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see http://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://doc.scrapy.org/en/latest/contributing.html#running-tests """ import os diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tests/keys/cert.pem b/tests/keys/cert.pem deleted file mode 100644 index 65478765e..000000000 --- a/tests/keys/cert.pem +++ /dev/null @@ -1,36 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzER -MA8GA1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQD -ExtNMkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5n -cHNAcG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzEL -MAkGA1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEB -BQADSwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh -5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQC -MAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRl -MB0GA1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7 -hyNp65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoT -CE0yQ3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlw -dG8gQ2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3Qx -LmNvbYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6 -BoJuVwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++ -7QGG/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JE -WUQ9Ho4EzbYCOQ== ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIBPAIBAAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh -5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAQJBAIqm/bz4NA1H++Vx5Ewx -OcKp3w19QSaZAwlGRtsUxrP7436QjnREM3Bm8ygU11BjkPVmtrKm6AayQfCHqJoT -ZIECIQDW0BoMoL0HOYM/mrTLhaykYAVqgIeJsPjvkEhTFXWBuQIhAM3deFAvWNu4 -nklUQ37XsCT2c9tmNt1LAT+slG2JOTTRAiAuXDtC/m3NYVwyHfFm+zKHRzHkClk2 -HjubeEgjpj32AQIhAJqMGTaZVOwevTXvvHwNEH+vRWsAYU/gbx+OQB+7VOcBAiEA -oolb6NMg/R3enNPvS1O4UU1H8wpaF77L4yiSWlE0p4w= ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE REQUEST----- -MIIBDTCBuAIBADBTMQswCQYDVQQGEwJTRzERMA8GA1UEChMITTJDcnlwdG8xEjAQ -BgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3DQEJARYObmdwc0Bwb3N0MS5jb20w -XDANBgkqhkiG9w0BAQEFAANLADBIAkEArL57d26W9fNXvOhNlZzlPOACmvwOZ5Ad -NgLzJ1/MfsQQJ7hHVeHmTAjM664V+fXvwUGJLziCeBo1ysWLRnl8CQIDAQABoAAw -DQYJKoZIhvcNAQEEBQADQQA7uqbrNTjVWpF6By5ZNPvhZ4YdFgkeXFVWi5ao/TaP -Vq4BG021fJ9nlHRtr4rotpgHDX1rr+iWeHKsx4+5DRSy ------END CERTIFICATE REQUEST----- \ No newline at end of file diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf index 8aa338cd5..1f9c25e43 100644 --- a/tests/keys/example-com.conf +++ b/tests/keys/example-com.conf @@ -1,4 +1,4 @@ -# this is copied from http://stackoverflow.com/a/27931596 +# this is copied from https://stackoverflow.com/a/27931596 [ req ] default_bits = 2048 default_keyfile = server-key.pem @@ -24,7 +24,7 @@ organizationName_default = Example, LLC # Use a friendly name here because its presented to the user. The server's DNS # names are placed in Subject Alternate Names. Plus, DNS names here is deprecated -# by both IETF and CA/Browser Forums. If you place a DNS name here, then you +# by both IETF and CA/Browser Forums. If you place a DNS name here, then you # must include the DNS name in the SAN too (otherwise, Chrome and others that # strictly follow the CA/Browser Baseline Requirements will fail). commonName = Common Name (e.g. server FQDN or YOUR name) diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README new file mode 100644 index 000000000..8e94e1217 --- /dev/null +++ b/tests/keys/localhost-ip.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 +Email Address []:. + diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt new file mode 100644 index 000000000..13c5b5bd6 --- /dev/null +++ b/tests/keys/localhost.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.gen.README b/tests/keys/localhost.gen.README new file mode 100644 index 000000000..19c29a725 --- /dev/null +++ b/tests/keys/localhost.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:localhost +Email Address []:. + diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt new file mode 100644 index 000000000..48d7bd9a3 --- /dev/null +++ b/tests/keys/localhost.ip.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key new file mode 100644 index 000000000..1e12c1255 --- /dev/null +++ b/tests/keys/localhost.ip.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq +-----END PRIVATE KEY----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key new file mode 100644 index 000000000..da975e6d3 --- /dev/null +++ b/tests/keys/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= +-----END PRIVATE KEY----- diff --git a/tests/mockserver.py b/tests/mockserver.py index e611cc3ec..f36ce3c44 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -5,12 +5,15 @@ from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource +from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper +from twisted.web.util import redirectTo from twisted.internet import reactor, ssl from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode @@ -118,6 +121,17 @@ class Echo(LeafResource): 'body': to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) + render_POST = render_GET + + +class RedirectTo(LeafResource): + + def render(self, request): + goto = getarg(request, b'goto', b'/') + # we force the body content, otherwise Twisted redirectTo() + # returns HTML with sample 3 text sample 3 repetition +sample 3 repetition with fragment inner tag href with whitespaces diff --git a/tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png b/tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png new file mode 100644 index 000000000..738f6ed41 Binary files /dev/null and b/tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png differ diff --git a/tests/sample_data/test_site/files/images/python-powered-h-50x65.png b/tests/sample_data/test_site/files/images/python-powered-h-50x65.png new file mode 100644 index 000000000..37f6b4719 Binary files /dev/null and b/tests/sample_data/test_site/files/images/python-powered-h-50x65.png differ diff --git a/tests/sample_data/test_site/files/images/scrapy.png b/tests/sample_data/test_site/files/images/scrapy.png new file mode 100644 index 000000000..66e86f567 Binary files /dev/null and b/tests/sample_data/test_site/files/images/scrapy.png differ diff --git a/tests/spiders.py b/tests/spiders.py index 1038b69de..7816bf7c7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -11,7 +11,12 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor -class MetaSpider(Spider): +class MockServerSpider(Spider): + def __init__(self, mockserver=None, *args, **kwargs): + super(MockServerSpider, self).__init__(*args, **kwargs) + self.mockserver = mockserver + +class MetaSpider(MockServerSpider): name = 'meta' @@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) self.start_urls = [url] def parse(self, response): @@ -55,7 +60,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b) + url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider): return self.errback_func(failure) -class DuplicateStartRequestsSpider(Spider): +class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True name = 'duplicatestartrequests' distinct_urls = 2 @@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = "http://localhost:8998/echo?headers=1&body=test%d" % i + url = self.mockserver.url("/echo?headers=1&body=test%d" % i) yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index fa0b48998..0eb1b7944 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -18,7 +18,7 @@ class TestCloseSpider(TestCase): def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') itemcount = crawler.stats.get_value('item_scraped_count') @@ -28,7 +28,7 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') pagecount = crawler.stats.get_value('response_received_count') @@ -38,7 +38,7 @@ class TestCloseSpider(TestCase): def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ @@ -50,7 +50,7 @@ class TestCloseSpider(TestCase): def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') stats = crawler.stats diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 7733e7180..10076bbca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertEquals(200, settingsdict[EXT_PATH]) + self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b6d6db9ee..66dd17110 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,6 +29,21 @@ class MySpider(scrapy.Spider): self.logger.debug('It Works!') return [scrapy.Item(), dict(foo='bar')] + def parse_request_with_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Does Not Work :(') + else: + self.logger.debug('It Works!') + + def parse_request_without_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{0}' @@ -84,6 +99,30 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + @defer.inlineCallbacks + def test_request_with_meta(self): + raw_json_string = '{"foo" : "baz"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--meta', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-m', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + + @defer.inlineCallbacks + def test_request_without_meta(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-c', 'parse_request_without_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 2789d207c..4ac7fb786 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -28,4 +28,4 @@ class VersionTest(ProcessTest, unittest.TestCase): self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', - 'Platform']) + 'cryptography', 'Platform']) diff --git a/tests/test_commands.py b/tests/test_commands.py index 922098668..7d9071b64 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import inspect import os import sys import subprocess @@ -17,6 +18,7 @@ from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class ProjectTest(unittest.TestCase): @@ -220,12 +222,43 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_run_fail_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + ret = proc.returncode + self.assertNotEqual(ret, 0) + + def test_run_good_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + ret = proc.returncode + self.assertEqual(ret, 0) + def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=('-s', 'LOG_LEVEL=INFO')) self.assertNotIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) + def test_runspider_dnscache_disabled(self): + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + log = self.get_log(""" +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + def parse(self, response): + return {'test': 'value'} +""", + args=('-s', 'DNSCACHE_ENABLED=False')) + print(log) + self.assertNotIn("DNSLookupError", log) + self.assertIn("INFO: Spider opened", log) + def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=1')) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3fc13eeb7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -26,7 +26,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks @@ -42,7 +42,7 @@ class CrawlTestCase(TestCase): def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2) + yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) @@ -53,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): crawler = self.runner.create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @@ -61,13 +61,13 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -77,14 +77,14 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/status?n=503") + yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:65432/status?n=503") + yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -92,14 +92,14 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./") + yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1) + yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1) + yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -121,7 +121,7 @@ class CrawlTestCase(TestCase): def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), # crawler.spider.seedsseen) @@ -130,10 +130,10 @@ class CrawlTestCase(TestCase): def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -160,7 +160,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) + yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) self.assertEqual(str(l).count("Got response 200"), 1) @defer.inlineCallbacks @@ -168,7 +168,7 @@ with multiples lines # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=0") + yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -176,7 +176,7 @@ with multiples lines # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=1") + yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(l) def _assert_retried(self, log): @@ -186,7 +186,7 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1) + req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) req1 = req0.replace() req2 = req0.replace(headers={'Referer': None}) req3 = req0.replace(headers={'Referer': 'http://example.com'}) @@ -194,7 +194,7 @@ with multiples lines req1.meta['next'] = req2 req2.meta['next'] = req3 crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0) + yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) @@ -220,7 +220,7 @@ with multiples lines est.append(get_engine_status(crawler.engine)) crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) self.assertEqual(s['engine.spider.name'], crawler.spider.name) @@ -244,7 +244,7 @@ with multiples lines raise TestError crawler = self.runner.create_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(), TestError) + yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -256,7 +256,7 @@ with multiples lines } crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) yield self.assertFailure( - self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @@ -264,13 +264,13 @@ with multiples lines def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: yield self.runner.join() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..d3b80f460 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,15 +1,22 @@ +import logging +import os +import tempfile import warnings import unittest +from twisted.internet import defer +import twisted.trial.unittest + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle - class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -74,6 +81,48 @@ class SpiderSettingsTestCase(unittest.TestCase): self.assertIn(AutoThrottle, enabled_exts) +class CrawlerLoggingTestCase(unittest.TestCase): + def test_no_root_handler_installed(self): + handler = get_scrapy_root_handler() + if handler is not None: + logging.root.removeHandler(handler) + + class MySpider(scrapy.Spider): + name = 'spider' + + crawler = Crawler(MySpider, {}) + assert get_scrapy_root_handler() is None + + def test_spider_custom_settings_log_level(self): + with tempfile.NamedTemporaryFile() as log_file: + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file.name, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + logged = log_file.read().decode('utf8') + self.assertNotIn('debug message', logged) + self.assertIn('info message', logged) + self.assertIn('warning message', logged) + self.assertIn('error message', logged) + self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1) + self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1) + self.assertEqual( + crawler.stats.get_value('log_count/INFO') - info_count, 1) + self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + + class SpiderLoaderWithWrongInterface(object): def unneeded_method(self): @@ -135,3 +184,62 @@ class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_None(self): runner = CrawlerProcess() self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + + +class ExceptionSpider(scrapy.Spider): + name = 'exception' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + raise ValueError('Exception in from_crawler method') + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful_for_several(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + self.assertEqual(runner.bootstrap_failed, True) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed_for_several(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + yield runner.crawl(NoRequestsSpider) + + self.assertEqual(runner.bootstrap_failed, True) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..c91be2c0c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,20 +6,22 @@ try: from unittest import mock except ImportError: import mock -import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath from twisted.internet import reactor, defer, error from twisted.web import server, static, util, resource +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource + PayloadResource from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -27,14 +29,15 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider -from scrapy.http import Request +from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse +from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured -from tests.mockserver import MockServer, ssl_context_factory +from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider class DummyDH(object): @@ -95,9 +98,9 @@ class FileTestCase(unittest.TestCase): def test_download(self): def _test(response): - self.assertEquals(response.url, request.url) - self.assertEquals(response.status, 200) - self.assertEquals(response.body, b'0123456789') + self.assertEqual(response.url, request.url) + self.assertEqual(response.status, 200) + self.assertEqual(response.body, b'0123456789') request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -118,6 +121,52 @@ class ContentLengthHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"content-length")[0] +class ChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + if hasattr(request.channel, 'loseConnection'): # twisted >=16.3.0 + request.channel.loseConnection() + else: + request.channel.transport.loseConnection() + request.finish() + + class EmptyContentTypeHeaderResource(resource.Resource): """ A testing resource which renders itself as the value of request body @@ -128,14 +177,24 @@ class EmptyContentTypeHeaderResource(resource.Resource): return request.content.read() +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + def response(): + for i in range(1024): + request.write(b"x" * 1024) + request.finish() + reactor.callLater(0, response) + return server.NOT_DONE_YET + + class HttpTestCase(unittest.TestCase): scheme = 'http' download_handler_cls = HTTPDownloadHandler # only used for HTTPS tests - keyfile = 'keys/cert.pem' - certfile = 'keys/cert.pem' + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' def setUp(self): self.tmpname = self.mktemp() @@ -149,8 +208,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"largechunkedfile", LargeChunkedFileResource()) + r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' @@ -178,28 +241,28 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'') + d.addCallback(self.assertEqual, b'') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d @defer.inlineCallbacks @@ -222,24 +285,24 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals( + self.assertEqual( response.body, to_bytes('%s:%d' % (self.host, self.portno))) - self.assertEquals(request.headers, {}) + self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): - self.assertEquals(response.body, b'example.com') - self.assertEquals(request.headers.get('Host'), b'example.com') + self.assertEqual(response.body, b'example.com') + self.assertEqual(request.headers.get('Host'), b'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'example.com') + d.addCallback(self.assertEqual, b'example.com') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -254,17 +317,28 @@ class HttpTestCase(unittest.TestCase): https://bugs.python.org/issue14721 """ def _test(response): - self.assertEquals(response.body, b'0') + self.assertEqual(response.body, b'0') request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) + def test_content_length_zero_bodyless_post_only_one(self): + def _test(response): + import json + headers = Headers(json.loads(response.text)['headers']) + contentlengths = headers.getlist('Content-Length') + self.assertEqual(len(contentlengths), 1) + self.assertEqual(contentlengths, [b"0"]) + + request = Request(self.getURL('echo'), method='POST') + return self.download_request(request, Spider('foo')).addCallback(_test) + def test_payload(self): body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, body) + d.addCallback(self.assertEqual, body) return d @@ -290,7 +364,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_response_class_choosing_request(self): @@ -300,7 +374,7 @@ class Http11TestCase(HttpTestCase): body = b'Some plain text\ndata with tabs\t and null bytes\0' def _test_type(response): - self.assertEquals(type(response), TextResponse) + self.assertEqual(type(response), TextResponse) request = Request(self.getURL('nocontenttype'), body=body) d = self.download_request(request, Spider('foo')) @@ -315,12 +389,31 @@ class Http11TestCase(HttpTestCase): # response body. (regardless of headers) d = self.download_request(request, Spider('foo', download_maxsize=10)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") yield d d = self.download_request(request, Spider('foo', download_maxsize=9)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY, mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + @defer.inlineCallbacks def test_download_with_maxsize_per_req(self): meta = {'download_maxsize': 2} @@ -338,9 +431,56 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo', download_maxsize=100)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d + def test_download_chunked_content(self): + request = Request(self.getURL('chunked')) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + request = Request(self.getURL(url)) + d = self.download_request(request, Spider('foo')) + + def checkDataLoss(failure): + if failure.check(ResponseFailed): + if any(r.check(_DataLoss) for r in failure.value.reasons): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss(self, url='broken'): + request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False}) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + download_handler = self.download_handler_cls(Settings({ + 'DOWNLOAD_FAIL_ON_DATALOSS': False, + })) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + class Https11TestCase(Http11TestCase): scheme = 'https' @@ -365,6 +505,19 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' +class Https11InvalidDNSPattern(Https11TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + super(Https11InvalidDNSPattern, self).setUp() + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -381,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000})) + yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) failure = crawler.spider.meta['failure'] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=Request(url='http://localhost:8998')) + yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] @@ -398,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase): def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider) body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', + request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] @@ -407,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase): if six.PY2: request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') + request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') @@ -457,9 +610,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'http://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'http://example.com') http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -467,9 +620,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy_https_noconnect(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'https://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) @@ -477,9 +630,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_without_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'/path/to/resource') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/path/to/resource') request = Request(self.getURL('path/to/resource')) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -687,9 +840,6 @@ class BaseFTPTestCase(unittest.TestCase): password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} - if six.PY3: - skip = "Twisted missing ftp support for PY3" - def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler @@ -700,8 +850,8 @@ class BaseFTPTestCase(unittest.TestCase): userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) @@ -736,8 +886,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'I have the power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']}) + self.assertEqual(r.body, b'I have the power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -749,8 +899,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'Moooooooooo power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']}) + self.assertEqual(r.body, b'Moooooooooo power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -763,7 +913,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = "/tmp/file.txt" + local_fname = b"/tmp/file.txt" meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -772,10 +922,10 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': ['17']}) + self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) - with open(local_fname) as f: - self.assertEqual(f.read(), "I have the power!") + with open(local_fname, "rb") as f: + self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test) @@ -810,8 +960,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): os.mkdir(self.directory) fp = FilePath(self.directory) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server for anonymous access realm = FTPRealm(anonymousRoot=self.directory) @@ -828,3 +978,69 @@ class AnonymousFTPTestCase(BaseFTPTestCase): def tearDown(self): shutil.rmtree(self.directory) + + +class DataURITestCase(unittest.TestCase): + + def setUp(self): + self.download_handler = DataURIDownloadHandler(Settings()) + self.download_request = self.download_handler.download_request + self.spider = Spider('foo') + + def test_response_attrs(self): + uri = "data:,A%20brief%20note" + + def _test(response): + self.assertEqual(response.url, uri) + self.assertFalse(response.headers) + + request = Request(uri) + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype_encoding(self): + def _test(response): + self.assertEqual(response.text, 'A brief note') + self.assertEqual(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEqual(response.encoding, "US-ASCII") + + request = Request("data:,A%20brief%20note") + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype(self): + def _test(response): + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEqual(response.encoding, "iso-8859-7") + + request = Request("data:;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_text_charset(self): + def _test(response): + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.body, b'\xbe\xd3\xbe') + self.assertEqual(response.encoding, "iso-8859-7") + + request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_mediatype_parameters(self): + def _test(response): + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEqual(response.encoding, "utf-8") + + request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' + 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' + ',%CE%8E%CE%A3%CE%8E') + return self.download_request(request, self.spider).addCallback(_test) + + def test_base64(self): + def _test(response): + self.assertEqual(response.text, 'Hello, world.') + + request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 26d9794b6..17801e502 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -36,7 +36,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') assert self.mw.process_request(req2, self.spider) is None - self.assertEquals(req2.headers.get('Cookie'), b"C1=value1") + self.assertEqual(req2.headers.get('Cookie'), b"C1=value1") def test_setting_false_cookies_enabled(self): self.assertRaises( @@ -131,12 +131,12 @@ class CookiesMiddlewareTest(TestCase): # check that cookies are merged back req = Request('http://scrapytest.org/mergeme') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') def test_complex_cookies(self): # merge some cookies into jar @@ -157,7 +157,7 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request('http://scrapytest.org/bar') self.mw.process_request(req, self.spider) - self.assertEquals(req.headers.get('Cookie'), b'C2=value2') + self.assertEqual(req.headers.get('Cookie'), b'C2=value2') # embed nothing for scrapytest.org/baz req = Request('http://scrapytest.org/baz') @@ -167,7 +167,7 @@ class CookiesMiddlewareTest(TestCase): def test_merge_request_cookies(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers) @@ -181,7 +181,7 @@ class CookiesMiddlewareTest(TestCase): def test_cookiejar_key(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers, request=req) @@ -193,7 +193,7 @@ class CookiesMiddlewareTest(TestCase): req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) assert self.mw.process_request(req3, self.spider) is None - self.assertEquals(req3.headers.get('Cookie'), b'galleta=dulce') + self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') headers = {'Set-Cookie': 'C2=value2; path=/'} res2 = Response('http://scrapytest.org/', headers=headers, request=req3) @@ -213,16 +213,16 @@ class CookiesMiddlewareTest(TestCase): req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') assert self.mw.process_request(req5_2, self.spider) is None - self.assertEquals(req5_2.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1') req5_3 = Request('http://scrapytest.org/some-redirected-path') assert self.mw.process_request(req5_3, self.spider) is None - self.assertEquals(req5_3.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1') #skip cookie retrieval for not http request req6 = Request('file:///scrapy/sometempfile') assert self.mw.process_request(req6, self.spider) is None - self.assertEquals(req6.headers.get('Cookie'), None) + self.assertEqual(req6.headers.get('Cookie'), None) def test_local_domain(self): request = Request("http://example-host/", cookies={'currencyCookie': 'USD'}) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 80efa83f9..6a31dfcf8 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -22,15 +22,15 @@ class TestDefaultHeadersMiddleware(TestCase): defaults, spider, mw = self.get_defaults_spider_mw() req = Request('http://www.scrapytest.org') mw.process_request(req, spider) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']} req = Request('http://www.scrapytest.org', headers=headers) - self.assertEquals(req.headers, bytes_headers) + self.assertEqual(req.headers, bytes_headers) mw.process_request(req, spider) defaults.update(bytes_headers) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 446a99f36..586bdc0d1 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -18,20 +18,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 180) + self.assertEqual(req.meta.get('download_timeout'), 180) def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 20.1) + self.assertEqual(req.meta.get('download_timeout'), 20.1) def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 2) + self.assertEqual(req.meta.get('download_timeout'), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -39,4 +39,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): mw.spider_opened(spider) req.meta['download_timeout'] = 1 assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 1) + self.assertEqual(req.meta.get('download_timeout'), 1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 425a5cc79..3381632b0 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -23,10 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') def test_auth_already_set(self): req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 12b69860a..22946b98c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest): (True, 203, {'Last-Modified': self.yesterday}), (True, 300, {'Last-Modified': self.yesterday}), (True, 301, {'Last-Modified': self.yesterday}), + (True, 308, {'Last-Modified': self.yesterday}), (True, 401, {'Last-Modified': self.yesterday}), (True, 404, {'Cache-Control': 'public, max-age=600'}), (True, 302, {'Expires': self.tomorrow}), diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7924fb3b5..0745c8dd3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -7,6 +7,8 @@ from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS +from scrapy.responsetypes import responsetypes +from scrapy.utils.gz import gunzip from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -152,15 +154,29 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_no_content_type_header(self): + headers = { + 'Content-Encoding': 'identity', + } + plainbody = b"""Some page""" + respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) + response = respcls("http://www.example.com/index", headers=headers, body=plainbody) + request = Request("http://www.example.com/index") + + newresponse = self.mw.process_response(request, response, self.spider) + assert isinstance(newresponse, respcls) + self.assertEqual(newresponse.body, plainbody) + self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_gzipped_contenttype(self): response = self._getresponse('gzip') response.headers['Content-Type'] = 'application/gzip' request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.headers['Content-Encoding'], b'gzip') - self.assertEqual(response.headers['Content-Type'], b'application/gzip') + self.assertIsNot(newresponse, response) + self.assertTrue(newresponse.body.startswith(b' + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""" + gz_file = GzipFile(fileobj=f, mode='wb') + gz_file.write(plainbody) + gz_file.close() + + # build a gzipped response body containing this gzipped file + r = BytesIO() + gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp.write(f.getvalue()) + gz_resp.close() + + response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertEqual(gunzip(newresponse.body), plainbody) def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') @@ -190,4 +248,4 @@ class HttpCompressionTest(TestCase): response = response.replace(body = None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) - self.assertEquals(response.body, b'') + self.assertEqual(response.body, b'') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index c77179ceb..537126613 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,7 @@ from scrapy.settings import Settings spider = Spider('foo') -class TestDefaultHeadersMiddleware(TestCase): +class TestHttpProxyMiddleware(TestCase): failureException = AssertionError @@ -28,17 +28,17 @@ class TestDefaultHeadersMiddleware(TestCase): crawler = Crawler(spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) - def test_no_enviroment_proxies(self): + def test_no_environment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta, {}) + self.assertEqual(req.url, url) + self.assertEqual(req.meta, {}) - def test_enviroment_proxies(self): + def test_environment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' os.environ.pop('file_proxy', None) @@ -48,41 +48,41 @@ class TestDefaultHeadersMiddleware(TestCase): ('https://e.com', https_proxy), ('file://tmp/a', None)]: req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta.get('proxy'), proxy) + self.assertEqual(req.url, url) + self.assertEqual(req.meta.get('proxy'), proxy) def test_proxy_precedence_meta(self): os.environ['http_proxy'] = 'https://proxy.com' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'}) def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -90,27 +90,27 @@ class TestDefaultHeadersMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') def test_proxy_already_seted(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' @@ -142,4 +142,4 @@ class TestDefaultHeadersMiddleware(TestCase): os.environ['no_proxy'] = '*' req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) + self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e8c92affa..74137b4cd 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase): req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority - def test_redirect_301(self): - def _test(method): - url = 'http://www.example.com/301' + def test_redirect_3xx_permanent(self): + def _test(method, status=301): + url = 'http://www.example.com/{}'.format(status) url2 = 'http://www.example.com/redirected' req = Request(url, method=method) - rsp = Response(url, headers={'Location': url2}, status=301) + rsp = Response(url, headers={'Location': url2}, status=status) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -42,6 +42,14 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=307) + _test('POST', status=307) + _test('HEAD', status=307) + + _test('GET', status=308) + _test('POST', status=308) + _test('HEAD', status=308) + def test_dont_redirect(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' @@ -158,7 +166,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) def test_utf8_location(self): req = Request('http://scrapytest.org/first') @@ -166,7 +174,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) class MetaRefreshMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e129b71f8..51b79b6c3 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(crawler) + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): @@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase): # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.crawler.stats.get_value('retry/max_reached') == 1 + assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2 + assert self.crawler.stats.get_value('retry/count') == 2 + def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, @@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase): req = Request('http://www.scrapytest.org/%s' % exc.__name__) self._test_retry_exception(req, exc('foo')) + stats = self.crawler.stats + assert stats.get_value('retry/max_reached') == len(exceptions) + assert stats.get_value('retry/count') == len(exceptions) * 2 + assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2 + def _test_retry_exception(self, req, exception): # first retry req = self.mw.process_exception(req, exception, self.spider) @@ -95,5 +104,83 @@ class RetryTest(unittest.TestCase): self.assertEqual(req, None) +class MaxRetryTimesTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) + self.mw.max_retry_times = 2 + self.invalid_url = 'http://www.scrapytest.org/invalid_url' + + def test_with_settings_zero(self): + + # SETTINGS: RETRY_TIMES = 0 + self.mw.max_retry_times = 0 + + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_zero(self): + + # SETTINGS: meta(max_retry_times) = 0 + meta_max_retry_times = 0 + + req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + + def test_without_metakey(self): + + # SETTINGS: RETRY_TIMES is NON-ZERO + self.mw.max_retry_times = 5 + + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_greater(self): + + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self.mw.max_retry_times = 2 + meta_max_retry_times = 3 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_lesser(self): + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 5 + meta_max_retry_times = 4 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_dont_retry(self): + + # SETTINGS: meta(max_retry_times) = 4 + meta_max_retry_times = 4 + + req = Request(self.invalid_url, meta= \ + {'max_retry_times': meta_max_retry_times, 'dont_retry': True}) + + self._test_retry(req, DNSLookupError('foo'), 0) + + + def _test_retry(self, req, exception, max_retry_times): + + for i in range(0, max_retry_times): + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + # discard it + req = self.mw.process_exception(req, exception, self.spider) + self.assertEqual(req, None) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 1e41fdace..a286764fd 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw('default_useragent') req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'default_useragent') + self.assertEqual(req.headers['User-Agent'], b'default_useragent') def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent @@ -34,7 +34,7 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'spider_useragent') + self.assertEqual(req.headers['User-Agent'], b'spider_useragent') def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') @@ -43,7 +43,7 @@ class UserAgentMiddlewareTest(TestCase): req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'header_useragent') + self.assertEqual(req.headers['User-Agent'], b'header_useragent') def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2d137edf4..0d9f1e83c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase): try: with MockServer() as s: runner = CrawlerRunner(Settings(defaults)) + spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) with open(res_name, 'rb') as f: @@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): for item in items: @@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): pass @@ -319,14 +318,14 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( - ('json', b'[\n\n]'), + ('json', b'[]'), ('jsonlines', b''), ('xml', b'\n'), ('csv', b''), ) for fmt, expctd in formats: - settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None} data = yield self.exported_no_data(settings) self.assertEqual(data, expctd) @@ -425,25 +424,177 @@ class FeedExportTest(unittest.TestCase): header = ['foo'] formats = { - 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), + 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - for format in formats: - settings = {'FEED_FORMAT': format} + for format, expected in formats.items(): + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None} data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) formats = { - 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), + 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - for format in formats: - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} + settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'} + for format, expected in formats.items(): + settings['FEED_FORMAT'] = format data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) + + @defer.inlineCallbacks + def test_export_indentation(self): + items = [ + {'foo': ['bar']}, + {'key': 'value'}, + ] + + test_cases = [ + # JSON + { + 'format': 'json', + 'indent': None, + 'expected': b'[{"foo": ["bar"]},{"key": "value"}]', + }, + { + 'format': 'json', + 'indent': -1, + 'expected': b"""[ +{"foo": ["bar"]}, +{"key": "value"} +]""", + }, + { + 'format': 'json', + 'indent': 0, + 'expected': b"""[ +{"foo": ["bar"]}, +{"key": "value"} +]""", + }, + { + 'format': 'json', + 'indent': 2, + 'expected': b"""[ +{ + "foo": [ + "bar" + ] +}, +{ + "key": "value" +} +]""", + }, + { + 'format': 'json', + 'indent': 4, + 'expected': b"""[ +{ + "foo": [ + "bar" + ] +}, +{ + "key": "value" +} +]""", + }, + { + 'format': 'json', + 'indent': 5, + 'expected': b"""[ +{ + "foo": [ + "bar" + ] +}, +{ + "key": "value" +} +]""", + }, + + # XML + { + 'format': 'xml', + 'indent': None, + 'expected': b""" +barvalue""", + }, + { + 'format': 'xml', + 'indent': -1, + 'expected': b""" + +bar +value +""", + }, + { + 'format': 'xml', + 'indent': 0, + 'expected': b""" + +bar +value +""", + }, + { + 'format': 'xml', + 'indent': 2, + 'expected': b""" + + + + bar + + + + value + +""", + }, + { + 'format': 'xml', + 'indent': 4, + 'expected': b""" + + + + bar + + + + value + +""", + }, + { + 'format': 'xml', + 'indent': 5, + 'expected': b""" + + + + bar + + + + value + +""", + }, + ] + + for row in test_cases: + settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} + data = yield self.exported_data(items, settings) + print(row['format'], row['indent']) + self.assertEqual(row['expected'], data) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 549f779d8..0a9ed500a 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase): self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assert_(self.wrapped.info() is self.wrapped) + self.assertIs(self.wrapped.info(), self.wrapped) def test_getheaders(self): self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7eadb874f..a042f03b6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase): h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' for k, v in h.iteritems(): - self.assert_(isinstance(k, bytes)) + self.assertIsInstance(k, bytes) for s in v: - self.assert_(isinstance(s, bytes)) + self.assertIsInstance(s, bytes) def test_eq(self): url = 'http://www.scrapy.org' @@ -235,6 +235,26 @@ class RequestTest(unittest.TestCase): self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + def test_callback_is_callable(self): + def a_function(): + pass + r = self.request_class('http://example.com') + self.assertIsNone(r.callback) + r = self.request_class('http://example.com', a_function) + self.assertIs(r.callback, a_function) + with self.assertRaises(TypeError): + self.request_class('http://example.com', 'a_function') + + def test_errback_is_callable(self): + def a_function(): + pass + r = self.request_class('http://example.com') + self.assertIsNone(r.errback) + r = self.request_class('http://example.com', a_function, errback=a_function) + self.assertIs(r.errback, a_function) + with self.assertRaises(TypeError): + self.request_class('http://example.com', a_function, errback='a_function') + class FormRequestTest(RequestTest): @@ -425,6 +445,17 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'one'], [b'1']) self.assertEqual(fs[b'two'], [b'2']) + def test_from_response_drop_params(self): + response = _buildresponse( + """ + + + """) + req = self.request_class.from_response(response, formdata={'two': None}) + fs = _qs(req) + self.assertEqual(fs[b'one'], [b'1']) + self.assertNotIn(b'two', fs) + def test_from_response_override_method(self): response = _buildresponse( ''' @@ -513,6 +544,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 924bb7979..820758dc9 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -155,6 +155,10 @@ class BaseResponseTest(unittest.TestCase): self._assert_followed_url(Link('http://example.com/foo'), 'http://example.com/foo') + def test_follow_None_url(self): + r = self.response_class("http://example.com") + self.assertRaises(ValueError, r.follow, None) + def test_follow_whitespace_url(self): self._assert_followed_url('foo ', 'http://example.com/foo%20') @@ -162,7 +166,6 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') - def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() @@ -273,7 +276,10 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.text, u'WORD\ufffd\ufffd') + self.assertIn(r6.text, { + u'WORD\ufffd\ufffd', # w3lib < 1.19.0 + u'WORD\ufffd', # w3lib >= 1.19.0 + }) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, @@ -392,6 +398,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' ] @@ -401,6 +408,13 @@ class TextResponseTest(BaseResponseTest): for sel, url in zip(sellist, urls): self._assert_followed_url(sel, url, response=resp) + # select elements + self._assert_followed_url( + Selector(text='').css('link')[0], + 'http://example.com/foo', + response=resp + ) + # href attributes should work for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: for sel, url in zip(sellist, urls): diff --git a/tests/test_item.py b/tests/test_item.py index 85a554de0..2c1eb0dd3 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ + return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 MyItem() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 340c64f35..1d7c4f311 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -13,6 +13,7 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None + escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') @@ -26,13 +27,19 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -41,6 +48,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): @@ -50,6 +58,27 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_with_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample', ), unique=False, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_no_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample',), unique=True, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): @@ -73,6 +102,8 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', + text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') @@ -276,13 +307,19 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -291,9 +328,10 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample2.jpg', text=u''), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index fef227aa1..1366971be 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -121,6 +121,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -142,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = SgmlLinkExtractor + escapes_whitespace = True def test_deny_extensions(self): html = """asd and """ @@ -190,6 +192,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(self.response), [Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'),]) diff --git a/tests/test_loader.py b/tests/test_loader.py index 2693a18d9..3b5714058 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value('name', [u'$10']) try: - float('$10') + float(u'$10') except Exception as e: expected_exc_str = str(e) @@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assert_(isinstance(proc(['hello', 'world']), six.text_type)) + self.assertIsInstance(proc(['hello', 'world']), six.text_type) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -490,21 +490,21 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertIs(l.selector, sel) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -526,7 +526,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//p/text()') @@ -552,7 +552,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//div/text()', re='ma') @@ -568,7 +568,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_css('name', 'p::text') @@ -606,7 +606,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)') diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 11fe7b653..94e6c9fde 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -64,5 +64,30 @@ class LoggingContribTest(unittest.TestCase): assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + +class LogFormatterSubclass(LogFormatter): + def crawled(self, request, response, spider): + kwargs = super(LogFormatterSubclass, self).crawled( + request, response, spider) + CRAWLEDMSG = ( + u"Crawled (%(status)s) %(request)s (referer: " + u"%(referer)s)%(flags)s" + ) + return { + 'level': kwargs['level'], + 'msg': CRAWLEDMSG, + 'args': kwargs['args'] + } + + +class LogformatterSubclassTest(LoggingContribTest): + def setUp(self): + self.formatter = LogFormatterSubclass() + self.spider = Spider('default') + + def test_flags_in_request(self): + pass + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py new file mode 100644 index 000000000..5985a6f3e --- /dev/null +++ b/tests/test_pipeline_crawl.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +import os +import shutil + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from w3lib.url import add_or_replace_parameter + +from scrapy.crawler import CrawlerRunner +from scrapy import signals +from tests.mockserver import MockServer +from tests.spiders import SimpleSpider + + +class MediaDownloadSpider(SimpleSpider): + name = 'mediadownload' + + def _process_url(self, url): + return url + + def parse(self, response): + self.logger.info(response.headers) + self.logger.info(response.text) + item = { + self.media_key: [], + self.media_urls_key: [ + self._process_url(response.urljoin(href)) + for href in response.xpath(''' + //table[thead/tr/th="Filename"] + /tbody//a/@href + ''').extract()], + } + yield item + + +class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): + name = 'brokenmedia' + + def _process_url(self, url): + return url + '.foo' + + +class RedirectedMediaDownloadSpider(MediaDownloadSpider): + name = 'redirectedmedia' + + def _process_url(self, url): + return add_or_replace_parameter( + self.mockserver.url('/redirect-to'), + 'goto', url) + + +class FileDownloadCrawlTestCase(TestCase): + pipeline_class = 'scrapy.pipelines.files.FilesPipeline' + store_setting_key = 'FILES_STORE' + media_key = 'files' + media_urls_key = 'file_urls' + expected_checksums = set([ + '5547178b89448faf0015a13f904c936e', + 'c2281c83670e31d8aaab7cb642b824db', + 'ed3f6538dc15d4d9179dae57319edc5f']) + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + # prepare a directory for storing files + self.tmpmediastore = self.mktemp() + os.mkdir(self.tmpmediastore) + self.settings = { + 'ITEM_PIPELINES': {self.pipeline_class: 1}, + self.store_setting_key: self.tmpmediastore, + } + self.runner = CrawlerRunner(self.settings) + self.items = [] + + def tearDown(self): + shutil.rmtree(self.tmpmediastore) + self.items = [] + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.items.append(item) + + def _create_crawler(self, spider_class, **kwargs): + crawler = self.runner.create_crawler(spider_class, **kwargs) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + return crawler + + def _assert_files_downloaded(self, items, logs): + self.assertEqual(len(items), 1) + self.assertIn(self.media_key, items[0]) + + # check that logs show the expected number of successful file downloads + file_dl_success = 'File (downloaded): Downloaded file from' + self.assertEqual(logs.count(file_dl_success), 3) + + # check that the images/files checksums are what we know they should be + if self.expected_checksums is not None: + checksums = set( + i['checksum'] + for item in items + for i in item[self.media_key]) + self.assertEqual(checksums, self.expected_checksums) + + # check that the image files where actually written to the media store + for item in items: + for i in item[self.media_key]: + self.assertTrue( + os.path.exists( + os.path.join(self.tmpmediastore, i['path']))) + + def _assert_files_download_failure(self, crawler, items, code, logs): + + # check that the item does NOT have the "images/files" field populated + self.assertEqual(len(items), 1) + self.assertIn(self.media_key, items[0]) + self.assertFalse(items[0][self.media_key]) + + # check that there was 1 successful fetch and 3 other responses with non-200 code + self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + + # check that logs do show the failure on the file downloads + file_dl_failure = 'File (code: %d): Error downloading file from' % code + self.assertEqual(logs.count(file_dl_failure), 3) + + # check that no files were written to the media store + self.assertEqual(os.listdir(self.tmpmediastore), []) + + @defer.inlineCallbacks + def test_download_media(self): + crawler = self._create_crawler(MediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key) + self._assert_files_downloaded(self.items, str(log)) + + @defer.inlineCallbacks + def test_download_media_wrong_urls(self): + crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key) + self._assert_files_download_failure(crawler, self.items, 404, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_default_failure(self): + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) + self._assert_files_download_failure(crawler, self.items, 302, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_allowed(self): + settings = dict(self.settings) + settings.update({'MEDIA_ALLOW_REDIRECTS': True}) + self.runner = CrawlerRunner(settings) + + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) + self._assert_files_downloaded(self.items, str(log)) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) + + +class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' + store_setting_key = 'IMAGES_STORE' + media_key = 'images' + media_urls_key = 'image_urls' + + # somehow checksums for images are different for Python 3.3 + expected_checksums = None diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e3ec04b8d..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,12 +11,13 @@ from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete +from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore from tests import mock @@ -375,6 +376,34 @@ class TestS3FilesStore(unittest.TestCase): self.assertEqual(key.content_type, 'image/png') +class TestGCSFilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_gcs_environ() + uri = os.environ.get('GCS_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No GCS URI available for testing") + data = b"TestGCSFilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = 'full/filename' + store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') + u = urlparse(uri) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + self.assertEqual(content, data) + self.assertEqual(blob.metadata, {'foo': 'bar'}) + self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) + self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 342f25ea9..03c6d8059 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -81,20 +81,28 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255) im = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEquals(thumbnail.mode, 'RGB') - self.assertEquals(thumbnail.size, (10, 10)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) class DeprecatedImagesPipeline(ImagesPipeline): diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index f30b4fea3..5f6a6d9e6 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -6,6 +6,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline @@ -22,10 +23,12 @@ def _mocked_download_func(request, info): class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline + settings = None def setUp(self): self.spider = Spider('media.com') - self.pipe = self.pipeline_class(download_func=_mocked_download_func) + self.pipe = self.pipeline_class(download_func=_mocked_download_func, + settings=Settings(self.settings)) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo @@ -82,6 +85,11 @@ class BaseMediaPipelineTestCase(unittest.TestCase): new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item + def test_modify_media_request(self): + request = Request('http://url') + self.pipe._modify_media_request(request) + assert request.meta == {'handle_httpstatus_all': True} + class MockedMediaPipeline(MediaPipeline): @@ -249,3 +257,61 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, 'ITSME')]) self.assertEqual(self.pipe._mockcalled, \ ['get_media_requests', 'media_to_download', 'item_completed']) + + +class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + + def _assert_request_no3xx(self, pipeline_class, settings): + pipe = pipeline_class(settings=Settings(settings)) + request = Request('http://url') + pipe._modify_media_request(request) + + self.assertIn('handle_httpstatus_list', request.meta) + for status, check in [ + (200, True), + + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True)]: + if check: + self.assertIn(status, request.meta['handle_httpstatus_list']) + else: + self.assertNotIn(status, request.meta['handle_httpstatus_list']) + + def test_standard_setting(self): + self._assert_request_no3xx( + MediaPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_standard_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_specific_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True + }) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0f06fd53d..ae1236bcb 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import time +from six.moves.urllib.parse import urlsplit, urlunsplit from threading import Thread from libmproxy import controller, proxy from netlib import http_auth @@ -17,7 +18,7 @@ from tests.mockserver import MockServer class HTTPSProxy(controller.Master, Thread): - def __init__(self, port): + def __init__(self): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread): server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, cacert = cert_path), - port) + 0) + self.server = server Thread.__init__(self) controller.Master.__init__(self, server) + def http_address(self): + return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): @@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy(8888) + + self._proxy = HTTPSProxy() self._proxy.start() + # Wait for the proxy to start. time.sleep(1.0) - os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = self._proxy.http_address() + os.environ['http_proxy'] = self._proxy.http_address() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase): def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) @defer.inlineCallbacks def test_https_noconnect(self): - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error(l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request("https://localhost:8999/echo") + request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) with LogCapture() as l: yield crawler.crawl(seed=request) @@ -94,14 +105,16 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): + print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) def _assert_got_tunnel_error(self, log): - self.assertEqual(str(log).count('TunnelError'), 1) + print(log) + self.assertIn('TunnelError', str(log)) diff --git a/tests/test_selector.py b/tests/test_selector.py index af0cc4de2..526660cc8 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -84,7 +84,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_badly_encoded_body(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index 371b8c1ac..929e0fea8 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -207,7 +207,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) @@ -234,7 +234,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html']) @@ -258,7 +258,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) @@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml @@ -344,6 +348,33 @@ Sitemap: /sitemap-relative-url.xml 'http://example.com/sitemap-uppercase.xml', 'http://www.example.com/sitemap-relative-url.xml']) + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + + spider.sitemap_alternate_links = True + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/italiano/']) + class DeprecationTest(unittest.TestCase): @@ -425,3 +456,17 @@ class DeprecationTest(unittest.TestCase): self.assertEqual(len(requests), 1) self.assertEqual(requests[0].url, 'http://example.com/foo') self.assertEqual(len(w), 1) + + +class NoParseMethodSpiderTest(unittest.TestCase): + + spider_class = Spider + + def test_undefined_parse_method(self): + spider = self.spider_class('example.com') + text = b'Random text' + resp = TextResponse(url="http://www.example.com/random_url", body=text) + + exc_msg = 'Spider.parse callback is not defined' + with self.assertRaisesRegexp(NotImplementedError, exc_msg): + spider.parse(resp) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b2ad93b3f..1cd59b99a 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -91,13 +91,71 @@ class SpiderLoaderTest(unittest.TestCase): self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + def test_bad_spider_modules_exception(self): + + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + self.assertRaises(ImportError, SpiderLoader.from_settings, settings) + def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: module = 'tests.test_spiderloader.test_spiders.doesnotexist' - settings = Settings({'SPIDER_MODULES': [module]}) + settings = Settings({'SPIDER_MODULES': [module], + 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() self.assertEqual(spiders, []) + +class DuplicateSpiderNameLoaderTest(unittest.TestCase): + + def setUp(self): + orig_spiders_dir = os.path.join(module_dir, 'test_spiders') + self.tmpdir = self.mktemp() + os.mkdir(self.tmpdir) + self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + shutil.copytree(orig_spiders_dir, self.spiders_dir) + sys.path.append(self.tmpdir) + self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + + def tearDown(self): + del sys.modules['test_spiders_xxx'] + sys.path.remove(self.tmpdir) + + def test_dupename_warning(self): + # copy 1 spider module so as to have duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 1) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider3'", msg) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) + + def test_multiple_dupename_warning(self): + # copy 2 spider modules so as to have duplicate spider name + # This should issue 2 warning, 1 for each duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 1) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider1'", msg) + self.assertIn("'spider2'", msg) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index a3cdc0114..3685d5a6f 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -25,18 +25,18 @@ class TestDepthMiddleware(TestCase): result = [Request('http://scrapytest.org')] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out, result) + self.assertEqual(out, result) rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) - self.assertEquals(rdc, 1) + self.assertEqual(rdc, 1) req.meta['depth'] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out2, []) + self.assertEqual(out2, []) rdm = self.stats.get_value('request_depth_max', spider=self.spider) - self.assertEquals(rdm, 1) + self.assertEqual(rdm, 1) def tearDown(self): self.stats.close_spider(self.spider, '') diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e1407e6b3..dacd0147f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,20 +11,21 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.spiders import MockServerSpider -class _HttpErrorSpider(Spider): +class _HttpErrorSpider(MockServerSpider): name = 'httperror' - start_urls = [ - "http://localhost:8998/status?n=200", - "http://localhost:8998/status?n=404", - "http://localhost:8998/status?n=402", - "http://localhost:8998/status?n=500", - ] bypass_status_codes = set() def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) + self.start_urls = [ + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), + ] self.failed = set() self.skipped = set() self.parsed = set() @@ -67,16 +68,16 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEquals([], + self.assertEqual([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider)) @@ -84,11 +85,11 @@ class TestHttpErrorMiddleware(TestCase): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) @@ -102,11 +103,11 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -117,14 +118,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) @@ -139,9 +140,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): @@ -152,7 +153,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) @@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: - yield crawler.crawl(bypass_status_codes={402}) + yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) self.assertEqual(crawler.spider.parsed, {'200', '402'}) self.assertEqual(crawler.spider.skipped, {'402'}) self.assertEqual(crawler.spider.failed, {'404', '500'}) @@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 37c3a450b..7e4af0d4c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -5,7 +5,9 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler +import warnings class TestOffsiteMiddleware(TestCase): @@ -37,7 +39,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, onsite_reqs) + self.assertEqual(out, onsite_reqs) class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -49,7 +51,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response('http://scrapytest.org') reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -67,4 +69,14 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response('http://scrapytest.org') reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) + + +class TestOffsiteMiddleware5(TestOffsiteMiddleware4): + + def test_get_host_regex(self): + self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.get_host_regex(self.spider) + assert issubclass(w[-1].category, URLWarning) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index bd7673efb..21439c20e 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,21 +1,874 @@ +from six.moves.urllib.parse import urlparse from unittest import TestCase +import warnings +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) + def test(self): + + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEqual(out[0].headers.get('Referer'), referrer) + + +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinStrictOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # downgrade: send nothing + ('https://example.com/page.html', 'http://scrapy.org', None), + + # upgrade: send origin + ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinStrictOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # downgrade + ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), + ('https://example4.com/page.html', 'http://not.example4.com/', None), + + # non-TLS to non-TLS + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + + # upgrade + ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), + ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), + + # Different protocols: send origin as referrer + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + + # TLS to non-TLS downgrade: send nothing + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + + +class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + + +class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +class CustomPythonOrgPolicy(ReferrerPolicy): + """ + A dummy policy that returns referrer as http(s)://python.org + depending on the scheme of the target URL. + """ + def referrer(self, response, request): + scheme = urlparse(request).scheme + if scheme == 'https': + return b'https://python.org/' + elif scheme == 'http': + return b'http://python.org/' + + +class TestSettingsCustomPolicy(TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), + + ] + +# --- Tests using Request meta dict to set policy +class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + + +class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaSettingFallback(TestCase): + + params = [ + ( + # When an unknown policy is referenced in Request.meta + # (here, a typo error), + # the policy defined in settings takes precedence + {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # same as above but with string value for settings policy + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # request meta references a wrong policy but it is set, + # so the Referrer-Policy header in response is not used, + # and the settings' policy is applied + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # here, request meta does not set the policy + # so response headers take precedence + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {}, + UnsafeUrlPolicy, + False + ), + ( + # here, request meta does not set the policy, + # but response headers also use an unknown policy, + # so the settings' policy is used + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unknown'}, + {}, + OriginWhenCrossOriginPolicy, + True + ) + ] + + def test(self): + + origin = 'http://www.scrapy.org' + target = 'http://www.example.com' + + for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + spider = Spider('foo') + mw = RefererMiddleware(Settings(settings)) + + response = Response(origin, headers=response_headers) + request = Request(target, meta=request_meta) + + with warnings.catch_warnings(record=True) as w: + policy = mw.policy(response, request) + self.assertIsInstance(policy, policy_class) + + if check_warning: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + + +class TestSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERRER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEqual(mw.default_policy, p) + + def test_valid_name_casevariants(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERRER_POLICY': s.upper()}) + mw = RefererMiddleware(settings) + self.assertEqual(mw.default_policy, p) + + def test_invalid_name(self): + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(RuntimeError): + mw = RefererMiddleware(settings) + + +class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + +class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + +class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + +class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + """ + The empty string means "no-referrer-when-downgrade" + """ + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': ''} + + +class TestReferrerOnRedirect(TestRefererMiddleware): + + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + # redirecting to non-secure URL + (301, 'http://scrapytest.org/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.com/2', + ( + # redirecting to non-secure URL: different origin + (301, 'http://scrapytest.com/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ] + + def setUp(self): + self.spider = Spider('foo') + settings = Settings(self.settings) + self.referrermw = RefererMiddleware(settings) + self.redirectmw = RedirectMiddleware(settings) + + def test(self): + + for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + response = self.get_response(parent) + request = self.get_request(target) + + out = list(self.referrermw.process_spider_output(response, [request], self.spider)) + self.assertEqual(out[0].headers.get('Referer'), init_referrer) + + for status, url in redirections: + response = Response(request.url, headers={'Location': url}, status=status) + request = self.redirectmw.process_response(request, response, self.spider) + self.referrermw.request_scheduled(request, self.spider) + + assert isinstance(request, Request) + self.assertEqual(request.headers.get('Referer'), final_referrer) + + +class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): + """ + No Referrer policy never sets the "Referer" header. + HTTP redirections should not change that. + """ + settings = {'REFERRER_POLICY': 'no-referrer'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + (301, 'http://scrapytest.org/3'), + ), + None, + None, + ), + ( 'https://scrapytest.org/1', + 'https://example.com/2', # different origin + ( + (301, 'http://scrapytest.com/3'), + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): + """ + Same Origin policy sends the full URL as "Referer" if the target origin + is the same as the parent response (same protocol, same domain, same port). + + HTTP redirections to a different domain or a lower secure level + should have the "Referer" removed. + """ + settings = {'REFERRER_POLICY': 'same-origin'} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting from secure to non-secure URL == different origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # different domain == different origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ] + + +class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): + """ + Strict Origin policy will always send the "origin" as referrer + (think of it as the parent URL without the path part), + unless the security level is lower and no "Referer" is sent. + + Redirections from secure to non-secure URLs should have the + "Referrer" header removed if necessary. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', + 'http://scrapytest.org/102', + ( + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/', # send origin + b'http://scrapytest.org/', # redirects to same origin: send origin + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: no referrer + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): no referrer + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # HTTPS all along, so origin referrer is kept as-is + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent. + + Redirections to a different origin should strip the "Referer" + to the parent origin. + """ + settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: send origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + b'https://scrapytest.org/', + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + b'https://scrapytest.org/', + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/301', + 'http://scrapytest.org/302', # TLS to non-TLS: send origin + ( + (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) + ), + b'https://scrapytest.org/', + b'https://scrapytest.org/', + ), + ] + + +class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Strict Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent... + Unless there's also a downgrade in security and then the "Referer" header + is not sent. + + Redirections to a different origin should strip the "Referer" to the parent origin, + and from https:// to http:// will remove the "Referer" header. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: do not send the "Referer" header + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ), + None, + None, + ), + ] diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index dca868ecf..a0aae0fdd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,5 +17,5 @@ class TestUrlLengthMiddleware(TestCase): mw = UrlLengthMiddleware(maxlength=25) spider = Spider('foo') out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEquals(out, [short_url_req]) + self.assertEqual(out, [short_url_req]) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..3ded5c027 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,8 +1,11 @@ +import pickle + from queuelib.tests import test_queue as t from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader +from scrapy.selector import Selector class TestItem(Item): name = Field() @@ -14,6 +17,24 @@ class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) +def nonserializable_object_test(self): + q = self.queue() + try: + pickle.dumps(lambda x: x) + except Exception: + # Trigger Twisted bug #7989 + import twisted.persisted.styles # NOQA + self.assertRaises(ValueError, q.push, lambda x: x) + else: + # Use a different unpickleable object + class A(object): pass + a = A() + a.__reduce__ = a.__reduce_ex__ = None + self.assertRaises(ValueError, q.push, a) + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -30,11 +51,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), {'a': 'dict'}) - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -110,11 +127,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), 'a') - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 052dde37f..22e39821c 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -6,7 +6,7 @@ class UrlparseTestCase(unittest.TestCase): def test_s3_url(self): p = urlparse('s3://bucket/key/name?param=value') - self.assertEquals(p.scheme, 's3') - self.assertEquals(p.hostname, 'bucket') - self.assertEquals(p.path, '/key/name') - self.assertEquals(p.query, 'param=value') + self.assertEqual(p.scheme, 's3') + self.assertEqual(p.hostname, 'bucket') + self.assertEqual(p.path, '/key/name') + self.assertEqual(p.query, 'param=value') diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 80f797227..5b83869b8 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import unittest +from collections import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude @@ -7,17 +8,62 @@ __doctests__ = ['scrapy.utils.datatypes'] class CaselessDictTest(unittest.TestCase): - def test_init(self): + def test_init_dict(self): seq = {'red': 1, 'black': 3} d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_pair_sequence(self): seq = (('red', 1), ('black', 3)) d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_mapping(self): + class MyMapping(Mapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + + def test_init_mutable_mapping(self): + class MyMutableMapping(MutableMapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, value): + self._d[key] = value + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMutableMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + def test_caseless(self): d = CaselessDict() d['key_Lower'] = 1 @@ -156,22 +202,22 @@ class SequenceExcludeTest(unittest.TestCase): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEquals([], are_not_in) + self.assertEqual([], are_not_in) are_not_in = [v for v in range(10, 20) if v in d] - self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index f49bbfafe..003bb9b02 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -89,7 +89,7 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(itergood(), errors.append)) self.assertEqual(out, list(range(10))) - self.failIf(errors) + self.assertFalse(errors) def test_iter_errback_bad(self): def iterbad(): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e3889a4..b2e8610f8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -252,8 +252,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..832253aa4 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -23,20 +23,20 @@ class UtilsMiscTestCase(unittest.TestCase): 'tests.test_utils_misc.test_walk_modules.mod.mod0', 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') expected = [ 'tests.test_utils_misc.test_walk_modules.mod', 'tests.test_utils_misc.test_walk_modules.mod.mod0', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') expected = [ 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) self.assertRaises(ImportError, walk_modules, 'nomodule999') @@ -51,7 +51,7 @@ class UtilsMiscTestCase(unittest.TestCase): 'testegg.spiders.b', 'testegg' ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) finally: sys.path.remove(egg) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 6b7fcd4c2..7e2caace8 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,14 +25,14 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEquals('.scrapy/somepath', data_path('somepath')) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('.scrapy/somepath', data_path('somepath')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) def test_data_path_inside_project(self): with inside_a_project() as proj_path: expected = os.path.join(proj_path, '.scrapy', 'somepath') - self.assertEquals( + self.assertEqual( os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 9a0cc975d..f6133657b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,7 +1,9 @@ +import gc import functools import operator import unittest from itertools import count +import platform import six from scrapy.utils.python import ( @@ -95,9 +97,9 @@ class UtilsPythonTestCase(unittest.TestCase): a = Obj() b = Obj() # no attributes given return False - self.failIf(equal_attributes(a, b, [])) + self.assertFalse(equal_attributes(a, b, [])) # not existent attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.x = 1 b.x = 1 @@ -106,7 +108,7 @@ class UtilsPythonTestCase(unittest.TestCase): b.y = 2 # obj1 has no attribute y - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.y = 2 # equal attributes @@ -114,7 +116,7 @@ class UtilsPythonTestCase(unittest.TestCase): a.y = 1 # differente attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) # test callable a.meta = {} @@ -132,7 +134,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, [compare_z, 'x'])) # fail z equality a.meta['z'] = 2 - self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): class _Weakme(object): pass @@ -144,6 +146,9 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertNotEqual(v, wk[_Weakme()]) self.assertEqual(v, wk[k]) del k + for _ in range(100): + if wk._weakdict: + gc.collect() self.assertFalse(len(wk._weakdict)) @unittest.skipUnless(six.PY2, "deprecated function") @@ -151,9 +156,9 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertIsNot(d, d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): @@ -161,17 +166,17 @@ class UtilsPythonTestCase(unittest.TestCase): d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertIsNot(d, d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertIsNot(d, d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): @@ -208,10 +213,19 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) self.assertEqual(get_func_args(object), []) - # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) - self.assertEqual(get_func_args(" ".join), []) - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + if platform.python_implementation() == 'CPython': + # TODO: how do we fix this to return the actual argument names? + self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(" ".join), []) + self.assertEqual(get_func_args(operator.itemgetter(2)), []) + else: + stripself = not six.PY2 # PyPy3 exposes them as methods + self.assertEqual( + get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, stripself), ['list']) + self.assertEqual( + get_func_args(operator.itemgetter(2), stripself), ['obj']) + def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 073baadc2..dcc070b8f 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -17,8 +17,8 @@ class RequestSerializationTest(unittest.TestCase): def test_all_attributes(self): r = Request("http://www.example.com", - callback='parse_item', - errback='handle_error', + callback=self.spider.parse_item, + errback=self.spider.handle_error, method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, @@ -27,7 +27,7 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, flags=['testFlag']) - self._assert_serializes_ok(r) + self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): r = Request("http://www.example.com", body=b"\xa3") diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b7de85049..62edd420d 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase): self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) - self.assert_(isinstance(result[0][1], Failure)) + self.assertIsInstance(result[0][1], Failure) self.assertEqual(result[1], (self.ok_handler, "OK")) dispatcher.disconnect(self.error_handler, signal=test_signal) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f46d1d927..c2b9fc176 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, parse_url) + add_http_if_no_scheme, guess_scheme, + parse_url, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -241,5 +242,171 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) +class StripUrl(unittest.TestCase): + + def test_noop(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html'), + 'http://www.example.com/index.html') + + def test_noop_query_string(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_fragments(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), + 'http://www.example.com/index.html?somekey=somevalue#section') + + def test_path(self): + for input_url, origin, output_url in [ + ('http://www.example.com/', + False, + 'http://www.example.com/'), + + ('http://www.example.com', + False, + 'http://www.example.com'), + + ('http://www.example.com', + True, + 'http://www.example.com/'), + ]: + self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + + def test_credentials(self): + for i, o in [ + ('http://username@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + ('https://username:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + + def test_credentials_encoded_delims(self): + for i, o in [ + # user: "username@" + # password: none + ('http://username%40@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + # user: "username:pass" + # password: "" + ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + # user: "me" + # password: "user@domain.com" + ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + + def test_default_ports_creds_off(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + ('http://username:password@www.example.com:8080/index.html#section', + 'http://www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i), o) + + def test_default_ports(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html', + 'http://username:password@www.example.com/index.html'), + + ('http://username:password@www.example.com:8080/index.html', + 'http://username:password@www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + + def test_default_ports_keep(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com:21/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + + def test_origin_only(self): + for i, o in [ + ('http://username:password@www.example.com/index.html', + 'http://www.example.com/'), + + ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', + 'http://www.example.com/'), + + ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', + 'http://www.example.com:8008/'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/'), + ]: + self.assertEqual(strip_url(i, origin_only=True), o) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3ad1aa70e..766329b57 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -71,7 +71,7 @@ class ParseUrlTestCase(unittest.TestCase): for url, test in tests: test = tuple( to_bytes(x) if not isinstance(x, int) else x for x in test) - self.assertEquals(client._parse(url), test, url) + self.assertEqual(client._parse(url), test, url) def test_externalUnicodeInterference(self): """ @@ -258,16 +258,16 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEquals, to_bytes(s)) + self.assertEqual, to_bytes(s)) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( - self.assertEquals, to_bytes("www.example.com"))]) + self.assertEqual, to_bytes("www.example.com"))]) def test_getPage(self): """ @@ -275,7 +275,7 @@ class WebClientTestCase(unittest.TestCase): the body of the response if the default method B{GET} is used. """ d = getPage(self.getURL("file")) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_getPageHead(self): @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) return d def test_timeoutTriggering(self): @@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assert_(b'404 - No Such Resource' in pageData) + self.assertIn(b'404 - No Such Resource', pageData) def testFactoryInfo(self): url = self.getURL('file') @@ -336,16 +336,16 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEquals(factory.status, b'200') - self.assert_(factory.version.startswith(b'HTTP/')) - self.assertEquals(factory.message, b'OK') - self.assertEquals(factory.response_headers[b'content-length'], b'10') + self.assertEqual(factory.status, b'200') + self.assertTrue(factory.version.startswith(b'HTTP/')) + self.assertEqual(factory.message, b'OK') + self.assertEqual(factory.response_headers[b'content-length'], b'10') def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEquals(pageData, + self.assertEqual(pageData, b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') @@ -360,6 +360,6 @@ class WebClientTestCase(unittest.TestCase): def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b'Content-Encoding']) - self.assertEquals(content_encoding, EncodingResource.out_encoding) - self.assertEquals( + self.assertEqual(content_encoding, EncodingResource.out_encoding) + self.assertEqual( response.body.decode(content_encoding), to_unicode(original_body)) diff --git a/tox.ini b/tox.ini index bbf50b733..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests +# Tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. @@ -8,9 +8,11 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore + google-cloud-storage Pillow != 3.0.0 leveldb -rtests/requirements.txt @@ -18,6 +20,8 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + GCS_TEST_FILE_URI + GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} @@ -38,6 +42,7 @@ deps = # https://packages.debian.org/en/jessie/zope/ basepython = python2.7 deps = + cryptography==0.6.1 pyOpenSSL==0.14 lxml==3.4.0 Twisted==14.0.2 @@ -59,31 +64,32 @@ basepython = pypy commands = py.test {posargs:scrapy tests} -[testenv:py33] -basepython = python3.3 +[testenv:py34] +basepython = python3.4 deps = -rrequirements-py3.txt # Extras Pillow -rtests/requirements-py3.txt -[testenv:py34] -basepython = python3.4 -deps = {[testenv:py33]deps} - [testenv:py35] basepython = python3.5 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} [testenv:py36] basepython = python3.6 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} + +[testenv:pypy3] +basepython = pypy3 +deps = {[testenv:py34]deps} +commands = + py.test {posargs:scrapy tests} [docs] changedir = docs deps = - Sphinx - sphinx_rtd_theme + -rdocs/requirements.txt [testenv:docs] changedir = {[docs]changedir}