diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 49a7e239f..57ff603fa 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.0dev2 +current_version = 1.3.0 commit = True tag = True tag_name = {new_version} diff --git a/.travis.yml b/.travis.yml index c58ab39a5..506f3779b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ branches: - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ env: - TOXENV=py27 - - TOXENV=precise + - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 - TOXENV=docs diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 95b4a7e3c..162602248 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,24 +1,41 @@ -# Contributor Code of Conduct +# Contributor Covenant Code of Conduct -As contributors and maintainers of this project, and in the interest of -fostering an open and welcoming community, we pledge to respect all people who -contribute through reporting issues, posting feature requests, updating -documentation, submitting pull requests or patches, and other activities. +## Our Pledge -We are committed to making participation in this project a harassment-free -experience for everyone, regardless of level of experience, gender, gender -identity and expression, sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, religion, or nationality. +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members Examples of unacceptable behavior by participants include: -* The use of sexualized language or imagery -* Personal attacks -* Trolling or insulting/derogatory comments +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment -* Publishing other's private information, such as physical or electronic - addresses, without explicit permission -* Other unethical or unprofessional conduct +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions @@ -26,25 +43,32 @@ that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. -By adopting this Code of Conduct, project maintainers commit themselves to -fairly and consistently applying these principles to every aspect of managing -this project. Project maintainers who do not follow or enforce the Code of -Conduct may be permanently removed from the project team. +## Scope This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting a project maintainer at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@scrapinghub.com. All complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. Maintainers are -obligated to maintain confidentiality with regard to the reporter of an -incident. +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.3.0, available at -[http://contributor-covenant.org/version/1/3/0/][version] +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/3/0/ +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/MANIFEST.in b/MANIFEST.in index 04b3e1fb9..94de4f3bf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,3 +12,4 @@ prune docs/build recursive-include extras * recursive-include bin * recursive-include tests * +global-exclude __pycache__ *.py[cod] diff --git a/README.rst b/README.rst index 74ae1f30d..38dda62e5 100644 --- a/README.rst +++ b/README.rst @@ -22,6 +22,10 @@ Scrapy :target: http://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report +.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg + :target: https://anaconda.org/conda-forge/scrapy + :alt: Conda Version + Overview ======== @@ -69,14 +73,17 @@ See http://scrapy.org/community/ Contributing ============ +See http://doc.scrapy.org/en/master/contributing.html + +Code of Conduct +--------------- + Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@scrapinghub.com. -See http://doc.scrapy.org/en/master/contributing.html - Companies using Scrapy ====================== diff --git a/docs/faq.rst b/docs/faq.rst index 415331515..ad11b071b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -317,6 +317,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/LIFO +.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/index.rst b/docs/index.rst index 4cb3eb741..289fb2b1b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,13 +13,15 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. +* Ask or search questions in `StackOverflow using the scrapy tag`_, * Search for information in the `archives of the scrapy-users mailing list`_, or `post a question`_. -* Ask a question in the `#scrapy IRC channel`_. +* Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. .. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues @@ -153,7 +155,6 @@ Solving specific problems topics/firebug topics/leaks topics/media-pipeline - topics/ubuntu topics/deploy topics/autothrottle topics/benchmarking @@ -186,9 +187,6 @@ Solving specific problems :doc:`topics/media-pipeline` Download files and/or images associated with your scraped items. -:doc:`topics/ubuntu` - Install latest Scrapy packages easily on Ubuntu - :doc:`topics/deploy` Deploying your Scrapy spiders and run them in a remote server. diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index c56348714..96363c7d5 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -5,21 +5,16 @@ Examples ======== The best way to learn is with examples, and Scrapy is no exception. For this -reason, there is an example Scrapy project named dirbot_, that you can use to -play and learn more about Scrapy. It contains the dmoz spider described in the -tutorial. +reason, there is an example Scrapy project named quotesbot_, that you can use to +play and learn more about Scrapy. It contains two spiders for +http://quotes.toscrape.com, one using CSS selectors and another one using XPath +expressions. -This dirbot_ project is available at: https://github.com/scrapy/dirbot - -It contains a README file with a detailed description of the project contents. +The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. +You can find more information about it in the project's README. If you're familiar with git, you can checkout the code. Otherwise you can -download a tarball or zip file of the project by clicking on `Downloads`_. +download the project as a zip file by clicking +`here `_. -The `scrapy tag on Snipplr`_ is used for sharing code snippets such as spiders, -middlewares, extensions, or scripts. Feel free (and encouraged!) to share any -code there. - -.. _dirbot: https://github.com/scrapy/dirbot -.. _Downloads: https://github.com/scrapy/dirbot/downloads -.. _scrapy tag on Snipplr: http://snipplr.com/all/tags/scrapy/ +.. _quotesbot: https://github.com/scrapy/quotesbot diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3364c3b31..767749ec5 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,48 +7,104 @@ Installation guide Installing Scrapy ================= -.. note:: Check :ref:`intro-install-platform-notes` first. +Scrapy runs on Python 2.7 and Python 3.3 or above +(except on Windows where Python 3 is not supported yet). -The installation steps assume that you have the following things installed: +If you’re already familiar with installation of Python packages, +you can install Scrapy and its dependencies from PyPI with:: -* `Python`_ 2.7 or above 3.3 + pip install Scrapy -* `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and - installs `setuptools`_ if not installed. Python 2.7.9 and later include - `pip`_ by default, so you may have it already. +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. -* `lxml`_. Most Linux distributions ships prepackaged versions of lxml. - Otherwise refer to http://lxml.de/installation.html +For more detailed and platform specifics instructions, read on. -* `OpenSSL`_. This comes preinstalled in all operating systems, except Windows - where the Python installer ships it bundled. -You can install Scrapy using pip (which is the canonical way to install Python -packages). To install using ``pip`` run:: +Things that are good to know +---------------------------- + +Scrapy is written in pure Python and depends on a few key Python packages (among others): + +* `lxml`_, an efficient XML and HTML parser +* `parsel`_, an HTML/XML data extraction library written on top of lxml, +* `w3lib`_, a multi-purpose helper for dealing with URLs and web page encodings +* `twisted`_, an asynchronous networking framework +* `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs + +The minimal versions which Scrapy is tested against are: + +* Twisted 14.0 +* lxml 3.4 +* pyOpenSSL 0.14 + +Scrapy may work with older versions of these packages +but it is not guaranteed it will continue working +because it’s not being tested against them. + +Some of these packages themselves depends on non-Python packages +that might require additional installation steps depending on your platform. +Please check :ref:`platform-specific guides below `. + +In case of any trouble related to these dependencies, +please refer to their respective installation instructions: + +* `lxml installation`_ +* `cryptography installation`_ + +.. _lxml installation: http://lxml.de/installation.html +.. _cryptography installation: https://cryptography.io/en/latest/installation/ + + +.. _intro-using-virtualenv: + +Using a virtual environment (recommended) +----------------------------------------- + +TL;DR: We recommend installing Scrapy inside a virtual environment +on all platforms. + +Python packages can be installed either globally (a.k.a system wide), +or in user-space. We do not recommend installing scrapy system wide. + +Instead, we recommend that you install scrapy within a so-called +"virtual environment" (`virtualenv`_). +Virtualenvs allow you to not conflict with already-installed Python +system packages (which could break some of your system tools and scripts), +and still install packages normally with ``pip`` (without ``sudo`` and the likes). + +To get started with virtual environments, see `virtualenv installation instructions`_. +To install it globally (having it globally installed actually helps here), +it should be a matter of running:: + + $ [sudo] pip install virtualenv + +Check this `user guide`_ on how to create your virtualenv. + +.. note:: + If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs. + +Once you have created a virtualenv, you can install scrapy inside it with ``pip``, +just like any other Python package. +(See :ref:`platform-specific guides ` +below for non-Python dependencies that you may need to install beforehand). + +Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. + +* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv. +* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv. + +.. _virtualenv: https://virtualenv.pypa.io +.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/ +.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html +.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/ - pip install Scrapy .. _intro-install-platform-notes: Platform specific installation notes ==================================== -Anaconda --------- - -.. note:: - - For Windows users, or if you have issues installing through `pip`, this is - the recommended way to install Scrapy. - -If you already have installed `Anaconda`_ or `Miniconda`_, the company -`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X. - -To install Scrapy using ``conda``, run:: - - conda install -c scrapinghub scrapy - - Windows ------- @@ -76,7 +132,7 @@ Windows * *(Only required for Python<2.7.9)* Install `pip`_ from https://pip.pypa.io/en/latest/installing/ - Now open a Command prompt to check ``pip`` is installed correctly:: + Now open a Command prompt to check ``pip`` is installed correctly:: pip --version @@ -89,37 +145,40 @@ Windows Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support Python 3 on Windows. -Ubuntu 9.10 or above --------------------- +Ubuntu 12.04 or above +--------------------- + +Scrapy is currently tested with recent-enough versions of lxml, +twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions. +But it should support older versions of Ubuntu too, like Ubuntu 12.04, +albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are typically too old and slow to catch up with latest Scrapy. -Instead, use the official :ref:`Ubuntu Packages `, which already -solve all dependencies for you and are continuously updated with the latest bug -fixes. -If you prefer to build the python dependencies locally instead of relying on -system packages you'll need to install their required non-python dependencies -first:: +To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install +these dependencies:: sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev -You can install Scrapy with ``pip`` after that:: +- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev`` + are required for ``lxml`` +- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography`` - pip install Scrapy +If you want to install scrapy on Python 3, you’ll also need Python 3 development headers:: + + sudo apt-get install python3 python3-dev + +Inside a :ref:`virtualenv `, +you can install Scrapy with ``pip`` after that:: + + pip install scrapy .. note:: - The same non-python dependencies can be used to install Scrapy in Debian Wheezy (7.0) and above. -Archlinux ---------- - -You can follow the generic instructions or install Scrapy from `AUR Scrapy package`:: - - yaourt -S scrapy Mac OS X -------- @@ -174,17 +233,39 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy + +Anaconda +-------- + + +Using Anaconda is an alternative to using a virtualenv and installing with ``pip``. + +.. note:: + + For Windows users, or if you have issues installing through ``pip``, this is + the recommended way to install Scrapy. + +If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ +community have up-to-date packages for Linux, Windows and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c conda-forge scrapy + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ -.. _easy_install: https://pypi.python.org/pypi/setuptools .. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx .. _lxml: http://lxml.de/ -.. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL +.. _parsel: https://pypi.python.org/pypi/parsel +.. _w3lib: https://pypi.python.org/pypi/w3lib +.. _twisted: https://twistedmatrix.com/ +.. _cryptography: https://cryptography.io/ +.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: http://brew.sh/ .. _zsh: http://www.zsh.org/ -.. _virtualenv: https://virtualenv.pypa.io/en/latest/ .. _Scrapinghub: http://scrapinghub.com .. _Anaconda: http://docs.continuum.io/anaconda/index .. _Miniconda: http://conda.pydata.org/docs/install/quick.html +.. _conda-forge: https://conda-forge.github.io/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6c1291c1f..7195017ff 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -19,74 +19,69 @@ Walk-through of an example spider In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. -So, here's the code for a spider that follows the links to the top -voted questions on StackOverflow and scrapes some data from each page:: +Here's the code for a spider that scrapes famous quotes from website +http://quotes.toscrape.com, following the pagination:: import scrapy - class StackOverflowSpider(scrapy.Spider): - name = 'stackoverflow' - start_urls = ['http://stackoverflow.com/questions?sort=votes'] + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/tag/humor/', + ] def parse(self, response): - for href in response.css('.question-summary h3 a::attr(href)'): - full_url = response.urljoin(href.extract()) - yield scrapy.Request(full_url, callback=self.parse_question) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } - def parse_question(self, response): - yield { - 'title': response.css('h1 a::text').extract_first(), - 'votes': response.css('.question .vote-count-post::text').extract_first(), - 'body': response.css('.question .post-text').extract_first(), - 'tags': response.css('.question .post-tag::text').extract(), - 'link': response.url, - } + next_page = response.css('li.next a::attr("href")').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Put this in a file, name it to something like ``stackoverflow_spider.py`` +Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json + scrapy runspider quotes_spider.py -o quotes.json -When this finishes you will have in the ``top-stackoverflow-questions.json`` file -a list of the most upvoted questions in StackOverflow in JSON format, containing the -title, link, number of upvotes, a list of the tags and the question content in HTML, -looking like this (reformatted for easier reading):: +When this finishes you will have in the ``quotes.json`` file a list of the +quotes in JSON format, containing text and author, looking like this (reformatted +here for better readability):: [{ - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array", - "tags": ["java", "c++", "performance", "optimization"], - "title": "Why is processing a sorted array faster than an unsorted array?", - "votes": "9924" + "author": "Jane Austen", + "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" }, { - "body": "... LONG HTML HERE ...", - "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule", - "tags": ["git", "git-submodules"], - "title": "How do I remove a Git submodule?", - "votes": "1764" + "author": "Groucho Marx", + "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" + }, + { + "author": "Steve Martin", + "text": "\u201cA day without sunshine is like, you know, night.\u201d" }, ...] - What just happened? ------------------- -When you ran the command ``scrapy runspider somefile.py``, Scrapy looked for a +When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked for a Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for StackOverflow top questions page) +attribute (in this case, only the URL for quotes in *humor* category) and called the default callback method ``parse``, passing the response object as -an argument. In the ``parse`` callback we extract the links to the -question pages using a CSS Selector with a custom extension that allows to get -the value for an attribute. Then we yield a few more requests to be sent, -registering the method ``parse_question`` as the callback to be called for each -of them as they finish. +an argument. In the ``parse`` callback, we loop through the quote elements +using a CSS Selector, yield a Python dict with the extracted quote text and author, +look for a link to the next page and schedule another request using the same +``parse`` method as callback. Here you notice one of the main advantages about Scrapy: requests are :ref:`scheduled and processed asynchronously `. This @@ -103,10 +98,6 @@ each request, limiting amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension ` that tries to figure out these automatically. -Finally, the ``parse_question`` callback scrapes the question data for each -page yielding a dict, which Scrapy then collects and writes to a JSON file as -requested in the command line. - .. note:: This is using :ref:`feed exports ` to generate the @@ -145,12 +136,13 @@ scraping easy and efficient, such as: :ref:`pipelines `). * Wide range of built-in extensions and middlewares for handling: - * cookies and session handling - * HTTP features like compression, authentication, caching - * user-agent spoofing - * robots.txt - * crawl depth restriction - * and more + + - cookies and session handling + - HTTP features like compression, authentication, caching + - user-agent spoofing + - robots.txt + - crawl depth restriction + - and more * A :ref:`Telnet console ` for hooking into a Python console running inside your Scrapy process, to introspect and debug your @@ -165,8 +157,8 @@ What's next? ============ The next steps for you are to :ref:`install Scrapy `, -:ref:`follow through the tutorial ` to learn how to organize -your code in Scrapy projects and `join the community`_. Thanks for your +:ref:`follow through the tutorial ` to learn how to create +a full-blown Scrapy project and `join the community`_. Thanks for your interest! .. _join the community: http://scrapy.org/community/ diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6ecd637c3..8e14d1b7c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,29 +7,36 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to use `Open directory project (dmoz) `_ as -our example domain to scrape. +We are going to scrape `quotes.toscrape.com `_, a website +that lists quotes from famous authors. This tutorial will walk you through these tasks: 1. Creating a new Scrapy project -2. Defining the Items you will extract -3. Writing a :ref:`spider ` to crawl a site and extract - :ref:`Items ` -4. Writing an :ref:`Item Pipeline ` to store the - extracted Items +2. Writing a :ref:`spider ` to crawl a site and extract data +3. Exporting the scraped data using the command line +4. Changing spider to recursively follow links +5. Using spider arguments Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of -Scrapy. If you're already familiar with other languages, and want to learn -Python quickly, we recommend `Learn Python The Hard Way`_. If you're new to programming -and want to start with Python, take a look at `this list of Python resources -for non-programmers`_. +Scrapy. + +If you're already familiar with other languages, and want to learn Python +quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, +you can follow the `Python Tutorial`_. + +If you're new to programming and want to start with Python, you may find useful +the online book `Learn Python The Hard Way`_. You can also take a look at `this +list of Python resources for non-programmers`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers +.. _Dive Into Python 3: http://www.diveintopython3.net +.. _Python Tutorial: https://docs.python.org/3/tutorial .. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ + Creating a project ================== @@ -46,7 +53,7 @@ This will create a ``tutorial`` directory with the following contents:: tutorial/ # project's Python module, you'll import your code from here __init__.py - items.py # project items file + items.py # project items definition file pipelines.py # project pipelines file @@ -54,208 +61,143 @@ This will create a ``tutorial`` directory with the following contents:: spiders/ # a directory where you'll later put your spiders __init__.py - ... -Defining our Item -================= - -`Items` are containers that will be loaded with the scraped data; they work -like simple Python dicts. While you can use plain Python dicts with Scrapy, -`Items` provide additional protection against populating undeclared fields, -preventing typos. They can also be used with :ref:`Item Loaders -`, a mechanism with helpers to conveniently populate `Items`. - -They are declared by creating a :class:`scrapy.Item ` class and defining -its attributes as :class:`scrapy.Field ` objects, much like in an ORM -(don't worry if you're not familiar with ORMs, you will see that this is an -easy task). - -We begin by modeling the item that we will use to hold the site's data obtained -from dmoz.org. As we want to capture the name, url and description of the -sites, we define fields for each of these three attributes. To do that, we edit -``items.py``, found in the ``tutorial`` directory. Our Item class looks like this:: - - import scrapy - - class DmozItem(scrapy.Item): - title = scrapy.Field() - link = scrapy.Field() - desc = scrapy.Field() - -This may seem complicated at first, but defining an item class allows you to use other handy -components and helpers within Scrapy. - Our first Spider ================ -Spiders are classes that you define and Scrapy uses to scrape information from a -domain (or group of domains). +Spiders are classes that you define and that Scrapy uses to scrape information +from a website (or a group of websites). They must subclass +:class:`scrapy.Spider` and define the initial requests to make, optionally how +to follow links in the pages, and how to parse the downloaded page content to +extract data. -They define an initial list of URLs to download, how to follow links, and how -to parse the contents of pages to extract :ref:`items `. - -To create a Spider, you must subclass :class:`scrapy.Spider -` and define some attributes: - -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be - unique, that is, you can't set the same name for different Spiders. - -* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the - Spider will begin to crawl from. The first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data - contained in the start URLs. - -* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will - be called with the downloaded :class:`~scrapy.http.Response` object of each - start URL. The response is passed to the method as the first and only - argument. - - This method is responsible for parsing the response data and extracting - scraped data (as scraped items) and more URLs to follow. - - The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing - the response and returning scraped data (as :class:`~scrapy.item.Item` - objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). - -This is the code for our first Spider; save it in a file named -``dmoz_spider.py`` under the ``tutorial/spiders`` directory:: +This is the code for our first Spider. Save it in a file named +``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: import scrapy - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] + + class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): - filename = response.url.split("/")[-2] + '.html' + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page with open(filename, 'wb') as f: f.write(response.body) + self.log('Saved file %s' % filename) -Crawling --------- + +As you can see, our Spider subclasses :class:`scrapy.Spider ` +and defines some attributes and methods: + +* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be + unique within a project, that is, you can't set the same name for different + Spiders. + +* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of + Requests (you can return a list of requests or write a generator function) + which the Spider will begin to crawl from. Subsequent requests will be + generated successively from these initial requests. + +* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle + the response downloaded for each of the requests made. The response parameter + is an instance of :class:`~scrapy.http.TextResponse` that holds + the page content and has further helpful methods to handle it. + + The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + the scraped data as dicts and also finding new URLs to + follow and creating new requests (:class:`~scrapy.http.Request`) from them. + +How to run our spider +--------------------- To put our spider to work, go to the project's top level directory and run:: - scrapy crawl dmoz + scrapy crawl quotes -This command runs the spider with name ``dmoz`` that we've just added, that -will send some requests for the ``dmoz.org`` domain. You will get an output +This command runs the spider with name ``quotes`` that we've just added, that +will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: - 2014-01-23 18:13:07-0400 [scrapy] INFO: Scrapy started (bot: tutorial) - 2014-01-23 18:13:07-0400 [scrapy] INFO: Optional features available: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Overridden settings: {} - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled extensions: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled downloader middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled spider middlewares: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Enabled item pipelines: ... - 2014-01-23 18:13:07-0400 [scrapy] INFO: Spider opened - 2014-01-23 18:13:08-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 18:13:09-0400 [scrapy] INFO: Closing spider (finished) + ... (omitted for brevity) + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html + 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html + 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) + ... +Now, check the files in the current directory. You should notice that two new +files have been created: *quotes-1.html* and *quotes-2.html*, with the content +for the respective URLs, as our ``parse`` method instructs. -.. note:: - At the end you can see a log line for each URL defined in ``start_urls``. - Because these URLs are the starting ones, they have no referrers, which is - shown at the end of the log line, where it says ``(referer: None)``. +.. note:: If you are wondering why we haven't parsed the HTML yet, hold + on, we will cover that soon. -Now, check the files in the current directory. You should notice two new files -have been created: *Books.html* and *Resources.html*, with the content for the respective -URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy creates :class:`scrapy.Request ` objects -for each URL in the ``start_urls`` attribute of the Spider, and assigns -them the ``parse`` method of the spider as their callback function. - -These Requests are scheduled, then executed, and :class:`scrapy.http.Response` -objects are returned and then fed back to the spider, through the -:meth:`~scrapy.spiders.Spider.parse` method. - -Extracting Items ----------------- - -Introduction to Selectors -^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to extract data from web pages. Scrapy uses a mechanism -based on `XPath`_ or `CSS`_ expressions called :ref:`Scrapy Selectors -`. For more information about selectors and other extraction -mechanisms see the :ref:`Selectors documentation `. - -.. _XPath: https://www.w3.org/TR/xpath -.. _CSS: https://www.w3.org/TR/selectors - -Here are some examples of XPath expressions and their meanings: - -* ``/html/head/title``: selects the ```` element, inside the ``<head>`` - element of an HTML document - -* ``/html/head/title/text()``: selects the text inside the aforementioned - ``<title>`` element. - -* ``//td``: selects all the ``<td>`` elements - -* ``//div[@class="mine"]``: selects all ``div`` elements which contain an - attribute ``class="mine"`` - -These are just a couple of simple examples of what you can do with XPath, but -XPath expressions are indeed much more powerful. To learn more about XPath, we -recommend `this tutorial to learn XPath through examples -<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how -to think in XPath" <http://plasmasturm.org/log/xpath101/>`_. - -.. note:: **CSS vs XPath:** you can go a long way extracting data from web pages - using only CSS selectors. However, XPath offers more power because besides - navigating the structure, it can also look at the content: you're - able to select things like: *the link that contains the text 'Next Page'*. - Because of this, we encourage you to learn about XPath even if you - already know how to construct CSS selectors. - -For working with CSS and XPath expressions, Scrapy provides -:class:`~scrapy.selector.Selector` class and convenient shortcuts to avoid -instantiating selectors yourself every time you need to select something from a -response. - -You can see selectors as objects that represent nodes in the document -structure. So, the first instantiated selectors are associated with the root -node, or the entire document. - -Selectors have four basic methods (click on the method to see the complete API -documentation): - -* :meth:`~scrapy.selector.Selector.xpath`: returns a list of selectors, each of - which represents the nodes selected by the xpath expression given as - argument. - -* :meth:`~scrapy.selector.Selector.css`: returns a list of selectors, each of - which represents the nodes selected by the CSS expression given as argument. - -* :meth:`~scrapy.selector.Selector.extract`: returns a unicode string with the - selected data. - -* :meth:`~scrapy.selector.Selector.re`: returns a list of unicode strings - extracted by applying the regular expression given as argument. +Scrapy schedules the :class:`scrapy.Request <scrapy.http.Request>` objects +returned by the ``start_requests`` method of the Spider. Upon receiving a +response for each one, it instantiates :class:`~scrapy.http.Response` objects +and calls the callback method associated with the request (in this case, the +``parse`` method) passing the response as argument. -Trying Selectors in the Shell -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +A shortcut to the start_requests method +--------------------------------------- +Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method +that generates :class:`scrapy.Request <scrapy.http.Request>` objects from URLs, +you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +with a list of URLs. This list will then be used by the default implementation +of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +for your spider:: -To illustrate the use of Selectors we're going to use the built-in :ref:`Scrapy -shell <topics-shell>`, which also requires `IPython <http://ipython.org/>`_ (an extended Python console) -installed on your system. + import scrapy -To start a shell, you must go to the project's top level directory and run:: - scrapy shell "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/" + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + + def parse(self, response): + page = response.url.split("/")[-2] + filename = 'quotes-%s.html' % page + with open(filename, 'wb') as f: + f.write(response.body) + +The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +of the requests for those URLs, even though we haven't explicitly told Scrapy +to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +default callback method, which is called for requests without an explicitly +assigned callback. + + +Extracting data +--------------- + +The best way to learn how to extract data with Scrapy is trying selectors +using the shell :ref:`Scrapy shell <topics-shell>`. Run:: + + scrapy shell 'http://quotes.toscrape.com/page/1/' .. note:: @@ -263,217 +205,339 @@ To start a shell, you must go to the project's top level directory and run:: command-line, otherwise urls containing arguments (ie. ``&`` character) will not work. -This is what the shell looks like:: + On Windows, use double quotes instead:: + + scrapy shell "http://quotes.toscrape.com/page/1/" + +You will see something like:: [ ... Scrapy log here ... ] - - 2014-01-23 17:11:42-0400 [scrapy] DEBUG: Crawled (200) <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://quotes.toscrape.com/page/1/> (referer: None) [s] Available Scrapy objects: - [s] crawler <scrapy.crawler.Crawler object at 0x3636b50> + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler <scrapy.crawler.Crawler object at 0x7fa91d888c90> [s] item {} - [s] request <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] settings <scrapy.settings.Settings object at 0x3fadc50> - [s] spider <Spider 'default' at 0x3cebf50> + [s] request <GET http://quotes.toscrape.com/page/1/> + [s] response <200 http://quotes.toscrape.com/page/1/> + [s] settings <scrapy.settings.Settings object at 0x7fa91d888c10> + [s] spider <DefaultSpider 'default' at 0x7fa91c8af990> [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser + >>> - In [1]: +Using the shell, you can try selecting elements using `CSS`_ with the response +object:: -After the shell loads, you will have the response fetched in a local -``response`` variable, so if you type ``response.body`` you will see the body -of the response, or you can type ``response.headers`` to see its headers. + >>> response.css('title') + [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape'>] -More importantly ``response`` has a ``selector`` attribute which is an instance of -:class:`~scrapy.selector.Selector` class, instantiated with this particular ``response``. -You can run queries on ``response`` by calling ``response.selector.xpath()`` or -``response.selector.css()``. There are also some convenience shortcuts like ``response.xpath()`` -or ``response.css()`` which map directly to ``response.selector.xpath()`` and -``response.selector.css()``. +The result of running ``response.css('title')`` is a list-like object called +:class:`~scrapy.selector.SelectorList`, which represents a list of +:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +and allow you to run further queries to fine-grain the selection or extract the +data. + +To extract the text from the title above, you can do:: + + >>> response.css('title::text').extract() + ['Quotes to Scrape'] + +There are two things to note here: one is that we've added ``::text`` to the +CSS query, to mean we want to select only the text elements directly inside +```` element. If we don't specify ``::text``, we'd get the full title +element, including its tags:: + + >>> response.css('title').extract() + ['<title>Quotes to Scrape'] + +The other thing is that the result of calling ``.extract()`` is a list, because +we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When +you know you just want the first result, as in this case, you can do:: + + >>> response.css('title::text').extract_first() + 'Quotes to Scrape' + +As an alternative, you could've written:: + + >>> response.css('title::text')[0].extract() + 'Quotes to Scrape' + +However, using ``.extract_first()`` avoids an ``IndexError`` and returns +``None`` when it doesn't find any element matching the selection. + +There's a lesson here: for most scraping code, you want it to be resilient to +errors due to things not being found on a page, so that even if some parts fail +to be scraped, you can at least get **some** data. + +Besides the :meth:`~scrapy.selector.Selector.extract` and +:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use +the :meth:`~scrapy.selector.Selector.re` method to extract using `regular +expressions`:: + + >>> response.css('title::text').re(r'Quotes.*') + ['Quotes to Scrape'] + >>> response.css('title::text').re(r'Q\w+') + ['Quotes'] + >>> response.css('title::text').re(r'(\w+) to (\w+)') + ['Quotes', 'Scrape'] + +In order to find the proper CSS selectors to use, you might find useful opening +the response page from the shell in your web browser using ``view(response)``. +You can use your browser developer tools or extensions like Firebug (see +sections about :ref:`topics-firebug` and :ref:`topics-firefox`). + +`Selector Gadget`_ is also a nice tool to quickly find CSS selector for +visually selected elements, which works in many browsers. + +.. _regular expressions: https://docs.python.org/3/library/re.html +.. _Selector Gadget: http://selectorgadget.com/ -So let's try it:: +XPath: a brief intro +^^^^^^^^^^^^^^^^^^^^ - In [1]: response.xpath('//title') - Out[1]: [Open Directory - Computers: Progr'>] - - In [2]: response.xpath('//title').extract() - Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [3]: response.xpath('//title/text()') - Out[3]: [] - - In [4]: response.xpath('//title/text()').extract() - Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - - In [5]: response.xpath('//title/text()').re('(\w+):') - Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] +Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: -Extracting the data -^^^^^^^^^^^^^^^^^^^ + >>> response.xpath('//title') + [] + >>> response.xpath('//title/text()').extract_first() + 'Quotes to Scrape' -Now, let's try to extract some real information from those pages. +XPath expressions are very powerful, and are the foundation of Scrapy +Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You +can see that if you read closely the text representation of the selector +objects in the shell. -You could type ``response.body`` in the console, and inspect the source code to -figure out the XPaths you need to use. However, inspecting the raw HTML code -there could become a very tedious task. To make it easier, you can -use Firefox Developer Tools or some Firefox extensions like Firebug. For more -information see :ref:`topics-firebug` and :ref:`topics-firefox`. +While perhaps not as popular as CSS selectors, XPath expressions offer more +power because besides navigating the structure, it can also look at the +content. Using XPath, you're able to select things like: *select the link +that contains the text "Next Page"*. This makes XPath very fitting to the task +of scraping, and we encourage you to learn XPath even if you already know how to +construct CSS selectors, it will make scraping much easier. -After inspecting the page source, you'll find that the web site's information -is inside a ``
    `` element, in fact the *second* ``
      `` element. +We won't cover much of XPath here, but you can read more about :ref:`using XPath +with Scrapy Selectors here `. To learn more about XPath, we +recommend `this tutorial to learn XPath through examples +`_, and `this tutorial to learn "how +to think in XPath" `_. -So we can select each ``
    • `` element belonging to the site's list with this -code:: +.. _XPath: https://www.w3.org/TR/xpath +.. _CSS: https://www.w3.org/TR/selectors - response.xpath('//ul/li') +Extracting quotes and authors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -And from them, the site's descriptions:: +Now that you know a bit about selection and extraction, let's complete our +spider by writing the code to extract the quotes from the web page. - response.xpath('//ul/li/text()').extract() +Each quote in http://quotes.toscrape.com is represented by HTML elements that look +like this: -The site's titles:: +.. code-block:: html - response.xpath('//ul/li/a/text()').extract() +
      + “The world as we have created it is a process of our + thinking. It cannot be changed without changing our thinking.” + + by Albert Einstein + (about) + +
      + Tags: + change + deep-thoughts + thinking + world +
      +
      -And the site's links:: +Let's open up scrapy shell and play a bit to find out how to extract the data +we want:: - response.xpath('//ul/li/a/@href').extract() + $ scrapy shell 'http://quotes.toscrape.com' -As we've said before, each ``.xpath()`` call returns a list of selectors, so we can -concatenate further ``.xpath()`` calls to dig deeper into a node. We are going to use -that property here, so:: +We get a list of selectors for the quote HTML elements with:: - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc + >>> response.css("div.quote") -.. note:: +Each of the selectors returned by the query above allows us to run further +queries over their sub-elements. Let's assign the first selector to a +variable, so that we can run our CSS selectors directly on a particular quote:: - For a more detailed description of using nested selectors, see - :ref:`topics-selectors-nesting-selectors` and - :ref:`topics-selectors-relative-xpaths` in the :ref:`topics-selectors` - documentation + >>> quote = response.css("div.quote")[0] -Let's add this code to our spider:: +Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +using the ``quote`` object we just created:: - import scrapy - - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] - start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" - ] - - def parse(self, response): - for sel in response.xpath('//ul/li'): - title = sel.xpath('a/text()').extract() - link = sel.xpath('a/@href').extract() - desc = sel.xpath('text()').extract() - print title, link, desc + >>> title = quote.css("span.text::text").extract_first() + >>> title + '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' + >>> author = quote.css("small.author::text").extract_first() + >>> author + 'Albert Einstein' -Now try crawling dmoz.org again and you'll see sites being printed -in your output. Run:: +Given that the tags are a list of strings, we can use the ``.extract()`` method +to get all of them:: - scrapy crawl dmoz + >>> tags = quote.css("div.tags a.tag::text").extract() + >>> tags + ['change', 'deep-thoughts', 'thinking', 'world'] -Using our item --------------- +Having figured out how to extract each bit, we can now iterate over all the +quotes elements and put them together into a Python dictionary:: -:class:`~scrapy.item.Item` objects are custom Python dicts; you can access the -values of their fields (attributes of the class we defined earlier) using the -standard dict syntax like:: + >>> for quote in response.css("div.quote"): + ... text = quote.css("span.text::text").extract_first() + ... author = quote.css("small.author::text").extract_first() + ... tags = quote.css("div.tags a.tag::text").extract() + ... print(dict(text=text, author=author, tags=tags)) + {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} + {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} + ... a few more of these, omitted for brevity + >>> - >>> item = DmozItem() - >>> item['title'] = 'Example title' - >>> item['title'] - 'Example title' +Extracting data in our spider +------------------------------ -So, in order to return the data we've scraped so far, the final code for our -Spider would be like this:: +Let's get back to our spider. Until now, it doesn't extract any data in +particular, just saves the whole HTML page to a local file. Let's integrate the +extraction logic above into our spider. + +A Scrapy spider typically generates many dictionaries containing the data +extracted from the page. To do that, we use the ``yield`` Python keyword +in the callback, as you can see below:: import scrapy - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", - "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', ] def parse(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() - yield item + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } -.. note:: You can find a fully-functional variant of this spider in the dirbot_ - project available at https://github.com/scrapy/dirbot +If you run this spider, it will output the extracted data with the log:: -Now crawling dmoz.org yields ``DmozItem`` objects:: + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.\n], - 'link': [u'http://gnosis.cx/TPiP/'], - 'title': [u'Text Processing in Python']} - [scrapy] DEBUG: Scraped from <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - {'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], - 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], - 'title': [u'XML Processing with Python']} + +.. _storing-data: + +Storing the scraped data +======================== + +The simplest way to store the scraped data is by using :ref:`Feed exports +`, with the following command:: + + scrapy crawl quotes -o quotes.json + +That will generate an ``quotes.json`` file containing all scraped items, +serialized in `JSON`_. + +For historic reasons, Scrapy appends to a given file instead of overwriting +its contents. If you run this command twice without removing the file +before the second time, you'll end up with a broken JSON file. + +You can also used other formats, like `JSON Lines`_:: + + scrapy crawl quotes -o quotes.jl + +The `JSON Lines`_ format is useful because it's stream-like, you can easily +append new records to it. It doesn't have the same problem of JSON when you run +twice. Also, as each record is a separate line, you can process big files +without having to fit everything in memory, there are tools like `JQ`_ to help +doing that at the command-line. + +In small projects (like the one in this tutorial), that should be enough. +However, if you want to perform more complex things with the scraped items, you +can write an :ref:`Item Pipeline `. A placeholder file +for Item Pipelines has been set up for you when the project is created, in +``tutorial/pipelines.py``. Though you don't need to implement any item +pipelines if you just want to store the scraped items. + +.. _JSON Lines: http://jsonlines.org +.. _JQ: https://stedolan.github.io/jq Following links =============== -Let's say, instead of just scraping the stuff in *Books* and *Resources* pages, -you want everything that is under the `Python directory -`_. +Let's say, instead of just scraping the stuff from the first two pages +from http://quotes.toscrape.com, you want quotes from all the pages in the website. -Now that you know how to extract data from a page, why not extract the links -for the pages you are interested, follow them and then extract the data you -want for all of them? +Now that you know how to extract data from pages, let's see how to follow links +from them. -Here is a modification to our spider that does just that:: +First thing is to extract the link to the page we want to follow. Examining +our page, we can see there is a link to the next page with the following +markup: + +.. code-block:: html + + + +We can try extracting it in the shell:: + + >>> response.css('li.next a').extract_first() + 'Next ' + +This gets the anchor element, but we want the attribute ``href``. For that, +Scrapy supports a CSS extension that let's you select the attribute contents, +like this:: + + >>> response.css('li.next a::attr(href)').extract_first() + '/page/2/' + +Let's see now our spider modified to recursively follow the link to the next +page, extracting data from it:: import scrapy - from tutorial.items import DmozItem - class DmozSpider(scrapy.Spider): - name = "dmoz" - allowed_domains = ["dmoz.org"] + class QuotesSpider(scrapy.Spider): + name = "quotes" start_urls = [ - "http://www.dmoz.org/Computers/Programming/Languages/Python/", + 'http://quotes.toscrape.com/page/1/', ] def parse(self, response): - for href in response.css("ul.directory.dir-col > li > a::attr('href')"): - url = response.urljoin(href.extract()) - yield scrapy.Request(url, callback=self.parse_dir_contents) + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } - def parse_dir_contents(self, response): - for sel in response.xpath('//ul/li'): - item = DmozItem() - item['title'] = sel.xpath('a/text()').extract() - item['link'] = sel.xpath('a/@href').extract() - item['desc'] = sel.xpath('text()').extract() - yield item + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) -Now the `parse()` method only extracts the interesting links from the page, -builds a full absolute URL using the `response.urljoin` method (since the links can -be relative) and yields new requests to be sent later, registering as callback -the method `parse_dir_contents()` that will ultimately scrape the data we want. + +Now, after extracting the data, the ``parse()`` method looks for the link to +the next page, builds a full absolute URL using the +:meth:`~scrapy.http.Response.urljoin` method (since the links can be +relative) and yields a new request to the next page, registering itself as +callback to handle the data extraction for the next page and to keep the +crawling going through all the pages. What you see here is Scrapy's mechanism of following links: when you yield a Request in a callback method, Scrapy will schedule that request to be sent @@ -483,55 +547,120 @@ Using this, you can build complex crawlers that follow links according to rules you define, and extract different kinds of data depending on the page it's visiting. -A common pattern is a callback method that extracts some items, looks for a link -to follow to the next page and then yields a `Request` with the same callback -for it:: - - def parse_articles_follow_next_page(self, response): - for article in response.xpath("//article"): - item = ArticleItem() - - ... extract article data here - - yield item - - next_page = response.css("ul.navigation > li.next-page > a::attr('href')") - if next_page: - url = response.urljoin(next_page[0].extract()) - yield scrapy.Request(url, self.parse_articles_follow_next_page) - -This creates a sort of loop, following all the links to the next page until it -doesn't find one -- handy for crawling blogs, forums and other sites with +In our example, it creates a sort of loop, following all the links to the next page +until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. -Another common pattern is to build an item with data from more than one page, +More examples and patterns +-------------------------- + +Here is another spider that illustrates callbacks and following links, +this time for scraping author information:: + + + import scrapy + + + class AuthorSpider(scrapy.Spider): + name = 'author' + + start_urls = ['http://quotes.toscrape.com/'] + + def parse(self, response): + # follow links to author pages + for href in response.css('.author+a::attr(href)').extract(): + yield scrapy.Request(response.urljoin(href), + callback=self.parse_author) + + # follow pagination links + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, callback=self.parse) + + def parse_author(self, response): + def extract_with_css(query): + return response.css(query).extract_first().strip() + + yield { + 'name': extract_with_css('h3.author-title::text'), + 'birthdate': extract_with_css('.author-born-date::text'), + 'bio': extract_with_css('.author-description::text'), + } + +This spider will start from the main page, it will follow all the links to the +authors pages calling the ``parse_author`` callback for each of them, and also +the pagination links with the ``parse`` callback as we saw before. + +The ``parse_author`` callback defines a helper function to extract and cleanup the +data from a CSS query and yields the Python dict with the author data. + +Another interesting thing this spider demonstrates is that, even if there are +many quotes from the same author, we don't need to worry about visiting the +same author page multiple times. By default, Scrapy filters out duplicated +requests to URLs already visited, avoiding the problem of hitting servers too +much because of a programming mistake. This can be configured by the setting +:setting:`DUPEFILTER_CLASS`. + +Hopefully by now you have a good understanding of how to use the mechanism +of following links and callbacks with Scrapy. + +As yet another example spider that leverages the mechanism of following links, +check out the :class:`~scrapy.spiders.CrawlSpider` class for a generic +spider that implements a small rules engine that you can use to write your +crawlers on top of it. + +Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks `. -.. note:: - As an example spider that leverages this mechanism, check out the - :class:`~scrapy.spiders.CrawlSpider` class for a generic spider - that implements a small rules engine that you can use to write your - crawlers on top of it. +Using spider arguments +====================== -Storing the scraped data -======================== +You can provide command line arguments to your spiders by using the ``-a`` +option when running them:: -The simplest way to store the scraped data is by using :ref:`Feed exports -`, with the following command:: + scrapy crawl quotes -o quotes-humor.json -a tag=humor - scrapy crawl dmoz -o items.json +These arguments are passed to the Spider's ``__init__`` method and become +spider attributes by default. -That will generate an ``items.json`` file containing all scraped items, -serialized in `JSON`_. +In this example, the value provided for the ``tag`` argument will be available +via ``self.tag``. You can use this to make your spider fetch only quotes +with a specific tag, building the URL based on the argument:: -In small projects (like the one in this tutorial), that should be enough. -However, if you want to perform more complex things with the scraped items, you -can write an :ref:`Item Pipeline `. As with Items, a -placeholder file for Item Pipelines has been set up for you when the project is -created, in ``tutorial/pipelines.py``. Though you don't need to implement any item -pipelines if you just want to store the scraped items. + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + + def start_requests(self): + url = 'http://quotes.toscrape.com/' + tag = getattr(self, 'tag', None) + if tag is not None: + url = url + 'tag/' + tag + yield scrapy.Request(url, self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small a::text').extract_first(), + } + + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + next_page = response.urljoin(next_page) + yield scrapy.Request(next_page, self.parse) + + +If you pass the ``tag=humor`` argument to this spider, you'll notice that it +will only visit URLs from the ``humor`` tag, such as +``http://quotes.toscrape.com/tag/humor``. + +You can :ref:`learn more about handling spider arguments here `. Next steps ========== @@ -540,9 +669,10 @@ This tutorial covered only the basics of Scrapy, but there's a lot of other features not mentioned here. Check the :ref:`topics-whatelse` section in :ref:`intro-overview` chapter for a quick overview of the most important ones. -Then, we recommend you continue by playing with an example project (see -:ref:`intro-examples`), and then continue with the section -:ref:`section-basics`. +You can continue from the section :ref:`section-basics` to know more about the +command-line tool, spiders, selectors and other things the tutorial hasn't covered like +modeling the scraped data. If you prefer to play with an example project, check +the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON .. _dirbot: https://github.com/scrapy/dirbot diff --git a/docs/news.rst b/docs/news.rst index 5395db8e3..cce46599b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,8 +3,193 @@ Release notes ============= -1.1.2 (2016-08-18) ------------------- +Scrapy 1.3.0 (2016-12-21) +------------------------- + +This release comes rather soon after 1.2.2 for one main reason: +it was found out that releases since 0.18 up to 1.2.2 (included) use +some backported code from Twisted (``scrapy.xlib.tx.*``), +even if newer Twisted modules are available. +Scrapy now uses ``twisted.web.client`` and ``twisted.internet.endpoints`` directly. +(See also cleanups below.) + +As it is a major change, we wanted to get the bug fix out quickly +while not breaking any projects using the 1.2 series. + +New Features +~~~~~~~~~~~~ + +- ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` + arguments (:issue:`2272`) +- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside + scrapy shell now follow HTTP redirections by default (:issue:`2290`); + See :command:`fetch` and :command:`shell` for details. +- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; + this is technically **backwards incompatible** so please check your log parsers. +- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``, + instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``); + this is **backwards incompatible** if you have log parsers expecting the short + logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES` + set to ``True``. + +Dependencies & Cleanups +~~~~~~~~~~~~~~~~~~~~~~~ + +- Scrapy now requires Twisted >= 13.1 which is the case for many Linux + distributions already. +- As a consequence, we got rid of ``scrapy.xlib.tx.*`` modules, which + copied some of Twisted code for users stuck with an "old" Twisted version +- ``ChunkedTransferMiddleware`` is deprecated and removed from the default + downloader middlewares. + + +Scrapy 1.2.2 (2016-12-06) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Fix a cryptic traceback when a pipeline fails on ``open_spider()`` (:issue:`2011`) +- Fix embedded IPython shell variables (fixing :issue:`396` that re-appeared + in 1.2.0, fixed in :issue:`2418`) +- A couple of patches when dealing with robots.txt: + + - handle (non-standard) relative sitemap URLs (:issue:`2390`) + - handle non-ASCII URLs and User-Agents in Python 2 (:issue:`2373`) + +Documentation +~~~~~~~~~~~~~ + +- Document ``"download_latency"`` key in ``Request``'s ``meta`` dict (:issue:`2033`) +- Remove page on (deprecated & unsupported) Ubuntu packages from ToC (:issue:`2335`) +- A few fixed typos (:issue:`2346`, :issue:`2369`, :issue:`2369`, :issue:`2380`) + and clarifications (:issue:`2354`, :issue:`2325`, :issue:`2414`) + +Other changes +~~~~~~~~~~~~~ + +- Advertize `conda-forge`_ as Scrapy's official conda channel (:issue:`2387`) +- More helpful error messages when trying to use ``.css()`` or ``.xpath()`` + on non-Text Responses (:issue:`2264`) +- ``startproject`` command now generates a sample ``middlewares.py`` file (:issue:`2335`) +- Add more dependencies' version info in ``scrapy version`` verbose output (:issue:`2404`) +- Remove all ``*.pyc`` files from source distribution (:issue:`2386`) + +.. _conda-forge: https://anaconda.org/conda-forge/scrapy + + +Scrapy 1.2.1 (2016-10-21) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Include OpenSSL's more permissive default ciphers when establishing + TLS/SSL connections (:issue:`2314`). +- Fix "Location" HTTP header decoding on non-ASCII URL redirects (:issue:`2321`). + +Documentation +~~~~~~~~~~~~~ + +- Fix JsonWriterPipeline example (:issue:`2302`). +- Various notes: :issue:`2330` on spider names, + :issue:`2329` on middleware methods processing order, + :issue:`2327` on getting multi-valued HTTP headers as lists. + +Other changes +~~~~~~~~~~~~~ + +- Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`). + + +Scrapy 1.2.0 (2016-10-03) +------------------------- + +New Features +~~~~~~~~~~~~ + +- New :setting:`FEED_EXPORT_ENCODING` setting to customize the encoding + used when writing items to a file. + This can be used to turn off ``\uXXXX`` escapes in JSON output. + This is also useful for those wanting something else than UTF-8 + for XML or CSV output (:issue:`2034`). +- ``startproject`` command now supports an optional destination directory + to override the default one based on the project name (:issue:`2005`). +- New :setting:`SCHEDULER_DEBUG` setting to log requests serialization + failures (:issue:`1610`). +- JSON encoder now supports serialization of ``set`` instances (:issue:`2058`). +- Interpret ``application/json-amazonui-streaming`` as ``TextResponse`` (:issue:`1503`). +- ``scrapy`` is imported by default when using shell tools (:command:`shell`, + :ref:`inspect_response `) (:issue:`2248`). + +Bug fixes +~~~~~~~~~ + +- DefaultRequestHeaders middleware now runs before UserAgent middleware + (:issue:`2088`). **Warning: this is technically backwards incompatible**, + though we consider this a bug fix. +- HTTP cache extension and plugins that use the ``.scrapy`` data directory now + work outside projects (:issue:`1581`). **Warning: this is technically + backwards incompatible**, though we consider this a bug fix. +- ``Selector`` does not allow passing both ``response`` and ``text`` anymore + (:issue:`2153`). +- Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`). +- Fix for an odd gzip decompression bug (:issue:`1606`). +- Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` + (:issue:`2225`). +- Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). +- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). + +Refactoring +~~~~~~~~~~~ + +- ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`). + +.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url + +Tests & Requirements +~~~~~~~~~~~~~~~~~~~~ + +Scrapy's new requirements baseline is Debian 8 "Jessie". It was previously +Ubuntu 12.04 Precise. +What this means in practice is that we run continuous integration tests +with these (main) packages versions at a minimum: +Twisted 14.0, pyOpenSSL 0.14, lxml 3.4. + +Scrapy may very well work with older versions of these packages +(the code base still has switches for older Twisted versions for example) +but it is not guaranteed (because it's not tested anymore). + +Documentation +~~~~~~~~~~~~~ + +- Grammar fixes: :issue:`2128`, :issue:`1566`. +- Download stats badge removed from README (:issue:`2160`). +- New scrapy :ref:`architecture diagram ` (:issue:`2165`). +- Updated ``Response`` parameters documentation (:issue:`2197`). +- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`). +- Add StackOverflow as a support channel (:issue:`2257`). + + +Scrapy 1.1.3 (2016-09-22) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Class attributes for subclasses of ``ImagesPipeline`` and ``FilesPipeline`` + work as they did before 1.1.1 (:issue:`2243`, fixes :issue:`2198`) + +Documentation +~~~~~~~~~~~~~ + +- :ref:`Overview ` and :ref:`tutorial ` + rewritten to use http://toscrape.com websites + (:issue:`2236`, :issue:`2249`, :issue:`2252`). + + +Scrapy 1.1.2 (2016-08-18) +------------------------- Bug fixes ~~~~~~~~~ @@ -17,8 +202,8 @@ Bug fixes (the regression was introduced in 1.1.1) -1.1.1 (2016-07-13) ------------------- +Scrapy 1.1.1 (2016-07-13) +------------------------- Bug fixes ~~~~~~~~~ @@ -69,8 +254,8 @@ Tests - Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`) -1.1.0 (2016-05-11) ------------------- +Scrapy 1.1.0 (2016-05-11) +------------------------- This 1.1 release brings a lot of interesting features and bug fixes: @@ -258,8 +443,8 @@ Bugfixes to same remote host (:issue:`1912`). -1.0.6 (2016-05-04) ------------------- +Scrapy 1.0.6 (2016-05-04) +------------------------- - FIX: RetryMiddleware is now robust to non-standard HTTP status codes (:issue:`1857`) - FIX: Filestorage HTTP cache was checking wrong modified time (:issue:`1875`) @@ -267,8 +452,8 @@ Bugfixes - DOC: Consistency in selectors examples (:issue:`1869`) -1.0.5 (2016-02-04) ------------------- +Scrapy 1.0.5 (2016-02-04) +------------------------- - FIX: [Backport] Ignore bogus links in LinkExtractors (fixes :issue:`907`, :commit:`108195e`) - TST: Changed buildbot makefile to use 'pytest' (:commit:`1f3d90a`) @@ -276,8 +461,8 @@ Bugfixes - DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`) -1.0.4 (2015-12-30) ------------------- +Scrapy 1.0.4 (2015-12-30) +------------------------- - Ignoring xlib/tx folder, depending on Twisted version. (:commit:`7dfa979`) - Run on new travis-ci infra (:commit:`6e42f0b`) @@ -328,14 +513,14 @@ Bugfixes - Small grammatical change (:commit:`8752294`) - Add openssl version to version command (:commit:`13c45ac`) -1.0.3 (2015-08-11) ------------------- +Scrapy 1.0.3 (2015-08-11) +------------------------- - add service_identity to scrapy install_requires (:commit:`cbc2501`) - Workaround for travis#296 (:commit:`66af9cd`) -1.0.2 (2015-08-06) ------------------- +Scrapy 1.0.2 (2015-08-06) +------------------------- - Twisted 15.3.0 does not raises PicklingError serializing lambda functions (:commit:`b04dd7d`) - Minor method name fix (:commit:`6f85c7f`) @@ -344,8 +529,8 @@ Bugfixes - Fixed typos (:commit:`a9ae7b0`) - Fix doc reference. (:commit:`7c8a4fe`) -1.0.1 (2015-07-01) ------------------- +Scrapy 1.0.1 (2015-07-01) +------------------------- - Unquote request path before passing to FTPClient, it already escape paths (:commit:`cc00ad2`) - include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`) @@ -354,8 +539,8 @@ Bugfixes - DOC remove version suffix from ubuntu package (:commit:`5303c66`) - DOC Update release date for 1.0 (:commit:`c89fa29`) -1.0.0 (2015-06-19) ------------------- +Scrapy 1.0.0 (2015-06-19) +------------------------- You will find a lot of new features and bugfixes in this major release. Make sure to check our updated :ref:`overview ` to get a glance of @@ -720,8 +905,8 @@ Code refactoring (:issue:`805`) - rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`) -0.24.6 (2015-04-20) -------------------- +Scrapy 0.24.6 (2015-04-20) +-------------------------- - encode invalid xpath with unicode_escape under PY2 (:commit:`07cb3e5`) - fix IPython shell scope issue and load IPython user config (:commit:`2c8e573`) @@ -730,8 +915,8 @@ Code refactoring - Converted sel.xpath() calls to response.xpath() in Extracting the data (:commit:`c2c6d15`) -0.24.5 (2015-02-25) -------------------- +Scrapy 0.24.5 (2015-02-25) +-------------------------- - Support new _getEndpoint Agent signatures on Twisted 15.0.0 (:commit:`540b9bc`) - DOC a couple more references are fixed (:commit:`b4c454b`) @@ -756,14 +941,14 @@ Code refactoring - Update request-response.rst (:commit:`3f3263d`) - SgmlLinkExtractor - fix for parsing tag with Unicode present (:commit:`49b40f0`) -0.24.4 (2014-08-09) -------------------- +Scrapy 0.24.4 (2014-08-09) +-------------------------- - pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68`) - scrapy bench needs scrapy.tests* (:commit:`d6cb999`) -0.24.3 (2014-08-09) -------------------- +Scrapy 0.24.3 (2014-08-09) +-------------------------- - no need to waste travis-ci time on py3 for 0.24 (:commit:`8e080c1`) - Update installation docs (:commit:`1d0c096`) @@ -795,23 +980,23 @@ Code refactoring - better testcase for settings.overrides.setdefault (:commit:`e22daaf`) - Using CRLF as line marker according to http 1.1 definition (:commit:`5ec430b`) -0.24.2 (2014-07-08) -------------------- +Scrapy 0.24.2 (2014-07-08) +-------------------------- - Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`) - there is not support for python3 yet (:commit:`3cd6146`) - Update python compatible version set to debian packages (:commit:`fa5d76b`) - DOC fix formatting in release notes (:commit:`c6a9e20`) -0.24.1 (2014-06-27) -------------------- +Scrapy 0.24.1 (2014-06-27) +-------------------------- - Fix deprecated CrawlerSettings and increase backwards compatibility with .defaults attribute (:commit:`8e3f20a`) -0.24.0 (2014-06-26) -------------------- +Scrapy 0.24.0 (2014-06-26) +-------------------------- Enhancements ~~~~~~~~~~~~ @@ -890,15 +1075,15 @@ Bugfixes - Testsuite doesn't require PIL anymore (:issue:`585`) -0.22.2 (released 2014-02-14) ----------------------------- +Scrapy 0.22.2 (released 2014-02-14) +----------------------------------- - fix a reference to unexistent engine.slots. closes #593 (:commit:`13c099a`) - downloaderMW doc typo (spiderMW doc copy remnant) (:commit:`8ae11bf`) - Correct typos (:commit:`1346037`) -0.22.1 (released 2014-02-08) ----------------------------- +Scrapy 0.22.1 (released 2014-02-08) +----------------------------------- - localhost666 can resolve under certain circumstances (:commit:`2ec2279`) - test inspect.stack failure (:commit:`cc3eda3`) @@ -926,8 +1111,8 @@ Bugfixes - fix 0.22.0 release date (:commit:`af0219a`) - fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`) -0.22.0 (released 2014-01-17) ----------------------------- +Scrapy 0.22.0 (released 2014-01-17) +----------------------------------- Enhancements ~~~~~~~~~~~~ @@ -970,20 +1155,20 @@ Fixes - Fix tests runner under pip 1.5 (:issue:`513`) - Fix logging error when spider name is unicode (:issue:`479`) -0.20.2 (released 2013-12-09) ----------------------------- +Scrapy 0.20.2 (released 2013-12-09) +----------------------------------- - Update CrawlSpider Template with Selector changes (:commit:`6d1457d`) - fix method name in tutorial. closes GH-480 (:commit:`b4fc359` -0.20.1 (released 2013-11-28) ----------------------------- +Scrapy 0.20.1 (released 2013-11-28) +----------------------------------- - include_package_data is required to build wheels from published sources (:commit:`5ba1ad5`) - process_parallel was leaking the failures on its internal deferreds. closes #458 (:commit:`419a780`) -0.20.0 (released 2013-11-08) ----------------------------- +Scrapy 0.20.0 (released 2013-11-08) +----------------------------------- Enhancements ~~~~~~~~~~~~ @@ -1070,15 +1255,15 @@ List of contributors sorted by number of commits:: 1 cacovsky 1 Berend Iwema -0.18.4 (released 2013-10-10) ----------------------------- +Scrapy 0.18.4 (released 2013-10-10) +----------------------------------- - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) - Fix start_requests laziness and early hangs (:commit:`89faf52`) -0.18.3 (released 2013-10-03) ----------------------------- +Scrapy 0.18.3 (released 2013-10-03) +----------------------------------- - fix regression on lazy evaluation of start requests (:commit:`12693a5`) - forms: do not submit reset inputs (:commit:`e429f63`) @@ -1086,14 +1271,14 @@ List of contributors sorted by number of commits:: - backport master fixes to json exporter (:commit:`cfc2d46`) - Fix permission and set umask before generating sdist tarball (:commit:`06149e0`) -0.18.2 (released 2013-09-03) ----------------------------- +Scrapy 0.18.2 (released 2013-09-03) +----------------------------------- - Backport `scrapy check` command fixes and backward compatible multi crawler process(:issue:`339`) -0.18.1 (released 2013-08-27) ----------------------------- +Scrapy 0.18.1 (released 2013-08-27) +----------------------------------- - remove extra import added by cherry picked changes (:commit:`d20304e`) - fix crawling tests under twisted pre 11.0.0 (:commit:`1994f38`) @@ -1111,8 +1296,8 @@ List of contributors sorted by number of commits:: - minor updates to 0.18 release notes (:commit:`c45e5f1`) - fix contributters list format (:commit:`0b60031`) -0.18.0 (released 2013-08-09) ----------------------------- +Scrapy 0.18.0 (released 2013-08-09) +----------------------------------- - Lot of improvements to testsuite run using Tox, including a way to test on pypi - Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) @@ -1203,8 +1388,8 @@ contributors sorted by number of commits:: 1 Berend Iwema -0.16.5 (released 2013-05-30) ----------------------------- +Scrapy 0.16.5 (released 2013-05-30) +----------------------------------- - obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`) - fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`) @@ -1212,8 +1397,8 @@ contributors sorted by number of commits:: - Find form nodes in invalid html5 documents (:commit:`e3d6945`) - Fix typo labeling attrs type bool instead of list (:commit:`a274276`) -0.16.4 (released 2013-01-23) ----------------------------- +Scrapy 0.16.4 (released 2013-01-23) +----------------------------------- - fixes spelling errors in documentation (:commit:`6d2b3aa`) - add doc about disabling an extension. refs #132 (:commit:`c90de33`) @@ -1224,8 +1409,8 @@ contributors sorted by number of commits:: - fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`) - Update docs/topics/commands.rst (:commit:`28eac7a`) -0.16.3 (released 2012-12-07) ----------------------------- +Scrapy 0.16.3 (released 2012-12-07) +----------------------------------- - Remove concurrency limitation when using download delays and still ensure inter-request delays are enforced (:commit:`487b9b5`) - add error details when image pipeline fails (:commit:`8232569`) @@ -1237,8 +1422,8 @@ contributors sorted by number of commits:: - Fixed docs typo in SpiderOpenCloseLogging example (:commit:`7184094`) -0.16.2 (released 2012-11-09) ----------------------------- +Scrapy 0.16.2 (released 2012-11-09) +----------------------------------- - scrapy contracts: python2.6 compat (:commit:`a4a9199`) - scrapy contracts verbose option (:commit:`ec41673`) @@ -1248,8 +1433,8 @@ contributors sorted by number of commits:: - Fix SpiderState bug in Windows platforms (:commit:`58998f4`) -0.16.1 (released 2012-10-26) ----------------------------- +Scrapy 0.16.1 (released 2012-10-26) +----------------------------------- - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`) - better backwards compatibility for scrapy.conf.settings (:commit:`3403089`) @@ -1259,8 +1444,8 @@ contributors sorted by number of commits:: - set release date for 0.16.0 in news (:commit:`e292246`) -0.16.0 (released 2012-10-18) ----------------------------- +Scrapy 0.16.0 (released 2012-10-18) +----------------------------------- Scrapy changes: @@ -1301,8 +1486,8 @@ Scrapy changes: - number received responses are now tracked through Scrapy stats (stat name: ``response_received_count``) - removed ``scrapy.log.started`` attribute -0.14.4 ------- +Scrapy 0.14.4 +------------- - added precise to supported ubuntu distros (:commit:`b7e46df`) - fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`) @@ -1310,8 +1495,8 @@ Scrapy changes: - replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`) - return trial status as bin/runtests.sh exit value. #118 (:commit:`b7b2e7f`) -0.14.3 ------- +Scrapy 0.14.3 +------------- - forgot to include pydispatch license. #118 (:commit:`fd85f9c`) - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`) @@ -1323,8 +1508,8 @@ Scrapy changes: - fixed minor defect in link extractors documentation (:commit:`ba14f38`) - removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`) -0.14.2 ------- +Scrapy 0.14.2 +------------- - move buffer pointing to start of file before computing checksum. refs #92 (:commit:`6a5bef2`) - Compute image checksum before persisting images. closes #92 (:commit:`9817df1`) @@ -1338,8 +1523,8 @@ Scrapy changes: - scrapyd: fixed documentation link (:commit:`2b4e4c3`) - extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`) -0.14.1 ------- +Scrapy 0.14.1 +------------- - extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`) - bumped version to 0.14.1 (:commit:`6cb9e1c`) @@ -1353,8 +1538,8 @@ Scrapy changes: - Avoid _disconnectedDeferred AttributeError exception in Twisted>=11.1.0 (:commit:`98f3f87`) - allow spider to set autothrottle max concurrency (:commit:`175a4b5`) -0.14 ----- +Scrapy 0.14 +----------- New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1421,8 +1606,8 @@ Code rearranged and removed - Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`) - Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept. -0.12 ----- +Scrapy 0.12 +----------- The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. @@ -1464,8 +1649,8 @@ Deprecated/obsoleted functionality - Deprecated ``queue`` command in favor of using Scrapyd ``schedule.json`` API. See also: Scrapyd changes - Removed the !LxmlItemLoader (experimental contrib which never graduated to main contrib) -0.10 ----- +Scrapy 0.10 +----------- The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. @@ -1537,8 +1722,8 @@ Changes to settings - Removed ``COMMANDS_SETTINGS_MODULE`` setting (#201) - Renamed ``REQUEST_HANDLERS`` to ``DOWNLOAD_HANDLERS`` and make download handlers classes (instead of functions) -0.9 ---- +Scrapy 0.9 +---------- The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. @@ -1577,8 +1762,8 @@ Changes to default settings - Changed default ``SCHEDULER_ORDER`` to ``DFO`` (:rev:`1939`) -0.8 ---- +Scrapy 0.8 +---------- The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. @@ -1623,8 +1808,8 @@ Backwards-incompatible changes - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121) - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`) -0.7 ---- +Scrapy 0.7 +---------- First release of Scrapy. diff --git a/docs/topics/_images/scrapy_architecture_02.png b/docs/topics/_images/scrapy_architecture_02.png index 80be02078..5fe393f22 100644 Binary files a/docs/topics/_images/scrapy_architecture_02.png and b/docs/topics/_images/scrapy_architecture_02.png differ diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d470a0d41..985cc0433 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -171,7 +171,8 @@ SpiderLoader API This class method is used by Scrapy to create an instance of the class. It's called with the current project settings, and it loads the spiders - found in the modules of the :setting:`SPIDER_MODULES` setting. + found recursively in the modules of the :setting:`SPIDER_MODULES` + setting. :param settings: project settings :type settings: :class:`~scrapy.settings.Settings` instance diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ba0e2c61c..4ac39ad2d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -12,24 +12,77 @@ Overview The following diagram shows an overview of the Scrapy architecture with its components and an outline of the data flow that takes place inside the system -(shown by the green arrows). A brief description of the components is included +(shown by the red arrows). A brief description of the components is included below with links for more detailed information about them. The data flow is also described below. +.. _data-flow: + +Data flow +========= + .. image:: _images/scrapy_architecture_02.png :width: 700 :height: 470 :alt: Scrapy architecture +The data flow in Scrapy is controlled by the execution engine, and goes like +this: + +1. The :ref:`Engine ` gets the initial Requests to crawl from the + :ref:`Spider `. + +2. The :ref:`Engine ` schedules the Requests in the + :ref:`Scheduler ` and asks for the + next Requests to crawl. + +3. The :ref:`Scheduler ` returns the next Requests + to the :ref:`Engine `. + +4. The :ref:`Engine ` sends the Requests to the + :ref:`Downloader `, passing through the + :ref:`Downloader Middlewares ` (see + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`). + +5. Once the page finishes downloading the + :ref:`Downloader ` generates a Response (with + that page) and sends it to the Engine, passing through the + :ref:`Downloader Middlewares ` (see + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`). + +6. The :ref:`Engine ` receives the Response from the + :ref:`Downloader ` and sends it to the + :ref:`Spider ` for processing, passing + through the :ref:`Spider Middleware ` (see + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input`). + +7. The :ref:`Spider ` processes the Response and returns + scraped items and new Requests (to follow) to the + :ref:`Engine `, passing through the + :ref:`Spider Middleware ` (see + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). + +8. The :ref:`Engine ` sends processed items to + :ref:`Item Pipelines `, then send processed Requests to + the :ref:`Scheduler ` and asks for possible next Requests + to crawl. + +9. The process repeats (from step 1) until there are no more requests from the + :ref:`Scheduler `. + Components ========== +.. _component-engine: + Scrapy Engine ------------- The engine is responsible for controlling the data flow between all components -of the system, and triggering events when certain actions occur. See the Data -Flow section below for more details. +of the system, and triggering events when certain actions occur. See the +:ref:`Data Flow ` section above for more details. + +.. _component-scheduler: Scheduler --------- @@ -37,19 +90,25 @@ Scheduler The Scheduler receives requests from the engine and enqueues them for feeding them later (also to the engine) when the engine requests them. +.. _component-downloader: + Downloader ---------- The Downloader is responsible for fetching web pages and feeding them to the engine which, in turn, feeds them to the spiders. +.. _component-spiders: + Spiders ------- Spiders are custom classes written by Scrapy users to parse responses and -extract items (aka scraped items) from them or additional URLs (requests) to +extract items (aka scraped items) from them or additional requests to follow. For more information see :ref:`topics-spiders`. +.. _component-pipelines: + Item Pipeline ------------- @@ -58,6 +117,8 @@ extracted (or scraped) by the spiders. Typical tasks include cleansing, validation and persistence (like storing the item in a database). For more information see :ref:`topics-item-pipeline`. +.. _component-downloader-middleware: + Downloader middlewares ---------------------- @@ -76,6 +137,8 @@ Use a Downloader middleware if you need to do one of the following: For more information see :ref:`topics-downloader-middleware`. +.. _component-spider-middleware: + Spider middlewares ------------------ @@ -93,39 +156,6 @@ Use a Spider middleware if you need to For more information see :ref:`topics-spider-middleware`. -Data flow -========= - -The data flow in Scrapy is controlled by the execution engine, and goes like -this: - -1. The Engine gets the first URLs to crawl from the Spider. - -2. The Engine schedules the URLs in the Scheduler as Requests and asks for the - next URLs to crawl. - -3. The Scheduler returns the next URLs to crawl to the Engine. - -4. The Engine sends the URLs to the Downloader, passing through the - Downloader Middleware (request direction). - -5. Once the page finishes downloading the Downloader generates a Response (with - that page) and sends it to the Engine, passing through the Downloader - Middleware (response direction). - -6. The Engine receives the Response from the Downloader and sends it to the - Spider for processing, passing through the Spider Middleware (input direction). - -7. The Spider processes the Response and returns scraped items and new Requests - (to follow) to the Engine, passing through the Spider Middleware - (output direction). - -8. The Engine sends processed items to Item Pipelines and processed Requests to - the Scheduler. - -9. The process repeats (from step 1) until there are no more requests from the - Scheduler. - Event-driven networking ======================= diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 632190067..99469ebf1 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -18,40 +18,66 @@ To run it use:: You should see an output like this:: - 2013-05-16 13:08:46-0300 [scrapy] INFO: Scrapy 0.17.0 started (bot: scrapybot) - 2013-05-16 13:08:47-0300 [scrapy] INFO: Spider opened - 2013-05-16 13:08:47-0300 [scrapy] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:48-0300 [scrapy] INFO: Crawled 74 pages (at 4440 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:49-0300 [scrapy] INFO: Crawled 143 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:50-0300 [scrapy] INFO: Crawled 210 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:51-0300 [scrapy] INFO: Crawled 274 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:52-0300 [scrapy] INFO: Crawled 343 pages (at 4140 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:53-0300 [scrapy] INFO: Crawled 410 pages (at 4020 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:54-0300 [scrapy] INFO: Crawled 474 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:55-0300 [scrapy] INFO: Crawled 538 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:56-0300 [scrapy] INFO: Crawled 602 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Closing spider (closespider_timeout) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Crawled 666 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) - 2013-05-16 13:08:57-0300 [scrapy] INFO: Dumping Scrapy stats: - {'downloader/request_bytes': 231508, - 'downloader/request_count': 682, - 'downloader/request_method_count/GET': 682, - 'downloader/response_bytes': 1172802, - 'downloader/response_count': 682, - 'downloader/response_status_count/200': 682, - 'finish_reason': 'closespider_timeout', - 'finish_time': datetime.datetime(2013, 5, 16, 16, 8, 57, 985539), - 'log_count/INFO': 14, - 'request_depth_max': 34, - 'response_received_count': 682, - 'scheduler/dequeued': 682, - 'scheduler/dequeued/memory': 682, - 'scheduler/enqueued': 12767, - 'scheduler/enqueued/memory': 12767, - 'start_time': datetime.datetime(2013, 5, 16, 16, 8, 47, 676539)} - 2013-05-16 13:08:57-0300 [scrapy] INFO: Spider closed (closespider_timeout) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Scrapy 1.2.2 started (bot: quotesbot) + 2016-12-16 21:18:48 [scrapy.utils.log] INFO: Overridden settings: {'CLOSESPIDER_TIMEOUT': 10, 'ROBOTSTXT_OBEY': True, 'SPIDER_MODULES': ['quotesbot.spiders'], 'LOGSTATS_INTERVAL': 1, 'BOT_NAME': 'quotesbot', 'LOG_LEVEL': 'INFO', 'NEWSPIDER_MODULE': 'quotesbot.spiders'} + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled extensions: + ['scrapy.extensions.closespider.CloseSpider', + 'scrapy.extensions.logstats.LogStats', + 'scrapy.extensions.telnet.TelnetConsole', + 'scrapy.extensions.corestats.CoreStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: + ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 'scrapy.downloadermiddlewares.stats.DownloaderStats'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: + ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 'scrapy.spidermiddlewares.depth.DepthMiddleware'] + 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled item pipelines: + [] + 2016-12-16 21:18:49 [scrapy.core.engine] INFO: Spider opened + 2016-12-16 21:18:49 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:50 [scrapy.extensions.logstats] INFO: Crawled 70 pages (at 4200 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:51 [scrapy.extensions.logstats] INFO: Crawled 134 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:52 [scrapy.extensions.logstats] INFO: Crawled 198 pages (at 3840 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:53 [scrapy.extensions.logstats] INFO: Crawled 254 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:54 [scrapy.extensions.logstats] INFO: Crawled 302 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:55 [scrapy.extensions.logstats] INFO: Crawled 358 pages (at 3360 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:56 [scrapy.extensions.logstats] INFO: Crawled 406 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:57 [scrapy.extensions.logstats] INFO: Crawled 438 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:58 [scrapy.extensions.logstats] INFO: Crawled 470 pages (at 1920 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:18:59 [scrapy.core.engine] INFO: Closing spider (closespider_timeout) + 2016-12-16 21:18:59 [scrapy.extensions.logstats] INFO: Crawled 518 pages (at 2880 pages/min), scraped 0 items (at 0 items/min) + 2016-12-16 21:19:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats: + {'downloader/request_bytes': 229995, + 'downloader/request_count': 534, + 'downloader/request_method_count/GET': 534, + 'downloader/response_bytes': 1565504, + 'downloader/response_count': 534, + 'downloader/response_status_count/200': 534, + 'finish_reason': 'closespider_timeout', + 'finish_time': datetime.datetime(2016, 12, 16, 16, 19, 0, 647725), + 'log_count/INFO': 17, + 'request_depth_max': 19, + 'response_received_count': 534, + 'scheduler/dequeued': 533, + 'scheduler/dequeued/memory': 533, + 'scheduler/enqueued': 10661, + 'scheduler/enqueued/memory': 10661, + 'start_time': datetime.datetime(2016, 12, 16, 16, 18, 49, 799869)} + 2016-12-16 21:19:00 [scrapy.core.engine] INFO: Spider closed (closespider_timeout) -That tells you that Scrapy is able to crawl about 3900 pages per minute in the +That tells you that Scrapy is able to crawl about 3000 pages per minute in the hardware where you run it. Note that this is a very simple spider intended to follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..6636c30cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,34 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them); + this only affects the URL you may pass as argument on the command line; + once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + # (only for the URL passed as command line argument) + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 31545d548..3b9a5335a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -27,7 +27,11 @@ The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to be overridden) and then sorted by order to get the final sorted list of enabled middlewares: the first middleware is the one closer to the engine and -the last is the one closer to the downloader. +the last is the one closer to the downloader. In other words, +the :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request` +method of each middleware will be invoked in increasing +middleware order (100, 200, 300, ...) and the :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` method +of each middleware will be invoked in decreasing order. To decide which order to assign to your middleware see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting and pick a value according to @@ -234,14 +238,14 @@ header) and all cookies received in responses (ie. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: - 2011-04-06 14:35:10-0300 [scrapy] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy] DEBUG: Sending cookies to: + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ Set-Cookie: ip_isocode=US Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy] DEBUG: Crawled (200) (referer: None) + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [...] @@ -653,16 +657,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_ - HttpProxyMiddleware ------------------- @@ -966,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 62ebc4c08..aac93a91a 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: list + :type to: str or list of str :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: list + :type cc: str or list of str :param body: the e-mail body :type body: str diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index f0d3e660d..efdd8c46b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -280,7 +280,7 @@ Whether to export empty feeds (ie. feeds with no items). FEED_STORAGES ------------- -Default:: ``{}`` +Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. @@ -314,7 +314,7 @@ can disable any of these backends by assigning ``None`` to their URI scheme in FEED_EXPORTERS -------------- -Default:: ``{}`` +Default: ``{}`` A dict containing additional exporters supported by your project. The keys are serialization formats and the values are paths to :ref:`Item exporter diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst index beda3b8db..0cf45861a 100644 --- a/docs/topics/firefox.rst +++ b/docs/topics/firefox.rst @@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one after applying some browser clean up and executing Javascript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things when working with Firefox and XPath: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index b9b4c2058..8c7aa361f 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -27,9 +27,10 @@ Each item pipeline component is a Python class that must implement the following .. method:: process_item(self, item, spider) - This method is called for every item pipeline component and must either return - a dict with data, :class:`~scrapy.item.Item` (or any descendant class) object - or raise a :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer + This method is called for every item pipeline component. :meth:`process_item` + must either: return a dict with data, return an :class:`~scrapy.item.Item` + (or any descendant class) object, return a `Twisted Deferred`_ or raise + :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. :param item: the item scraped @@ -66,6 +67,8 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object +.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Item pipeline example ===================== @@ -103,9 +106,12 @@ format:: class JsonWriterPipeline(object): - def __init__(self): + def open_spider(self, spider): self.file = open('items.jl', 'wb') + def close_spider(self, spider): + self.file.close() + def process_item(self, item, spider): line = json.dumps(dict(item)) + "\n" self.file.write(line) @@ -123,14 +129,7 @@ MongoDB address and database name are specified in Scrapy settings; MongoDB collection is named after item class. The main point of this example is to show how to use :meth:`from_crawler` -method and how to clean up the resources properly. - -.. note:: - - Previous example (JsonWriterPipeline) doesn't clean up resources properly. - Fixing it is left as an exercise for the reader. - -:: +method and how to clean up the resources properly.:: import pymongo @@ -163,6 +162,55 @@ method and how to clean up the resources properly. .. _MongoDB: https://www.mongodb.org/ .. _pymongo: https://api.mongodb.org/python/current/ + +Take screenshot of item +----------------------- + +This example demonstrates how to return Deferred_ from :meth:`process_item` method. +It uses Splash_ to render screenshot of item url. Pipeline +makes request to locally running instance of Splash_. After request is downloaded +and Deferred callback fires, it saves item to a file and adds filename to an item. + +:: + + import scrapy + import hashlib + from urllib.parse import quote + + + class ScreenshotPipeline(object): + """Pipeline that uses Splash to render screenshot of + every Scrapy item.""" + + SPLASH_URL = "http://localhost:8050/render.png?url={}" + + def process_item(self, item, spider): + encoded_item_url = quote(item["url"]) + screenshot_url = self.SPLASH_URL.format(encoded_item_url) + request = scrapy.Request(screenshot_url) + dfd = spider.crawler.engine.download(request, spider) + dfd.addBoth(self.return_item, item) + return dfd + + def return_item(self, response, item): + if response.status != 200: + # Error happened, return item. + return item + + # Save screenshot to file, filename will be hash of url. + url = item["url"] + url_hash = hashlib.md5(url.encode("utf8")).hexdigest() + filename = "{}.png".format(url_hash) + with open(filename, "wb") as f: + f.write(response.body) + + # Store filename in item. + item["screenshot_filename"] = filename + return item + +.. _Splash: http://splash.readthedocs.io/en/stable/ +.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html + Duplicates filter ----------------- diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index b7aa6d985..ac3b614fc 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -10,7 +10,7 @@ Logging about the new logging system. Scrapy uses `Python's builtin logging system -`_ for event logging. We'll +`_ for event logging. We'll provide some simple examples to get you started, but for more advanced use-cases it's strongly suggested to read thoroughly its documentation. @@ -150,6 +150,7 @@ These settings can be used to configure the logging: * :setting:`LOG_FORMAT` * :setting:`LOG_DATEFORMAT` * :setting:`LOG_STDOUT` +* :setting:`LOG_SHORT_NAMES` The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be @@ -170,6 +171,10 @@ listed in `logging's logrecord attributes docs `_ respectively. +If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy +component that prints the log. It is unset by default, hence logs contain the +scrapy component responsible for that log output. + Command-line options -------------------- @@ -188,6 +193,43 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +Advanced customization +---------------------- + +Because Scrapy uses stdlib logging module, you can customize logging using +all features of stdlib logging. + +For example, let's say you're scraping a website which returns many +HTTP 404 and 500 responses, and you want to hide all messages like this:: + + 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring + response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + is not handled or not allowed + +The first thing to note is a logger name - it is in brackets: +``[scrapy.spidermiddlewares.httperror]``. If you get just ``[scrapy]`` then +:setting:`LOG_SHORT_NAMES` is likely set to True; set it to False and re-run +the crawl. + +Next, we can see that the message has INFO level. To hide it +we should set logging level for ``scrapy.spidermiddlewares.httperror`` +higher than INFO; next level after INFO is WARNING. It could be done +e.g. in the spider's ``__init__`` method:: + + import logging + import scrapy + + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('scrapy.spidermiddlewares.httperror') + logger.setLevel(logging.WARNING) + super().__init__(*args, **kwargs) + +If you run this spider again then INFO messages from +``scrapy.spidermiddlewares.httperror`` logger will be gone. + scrapy.utils.log module ======================= diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f13a72a0b..82c0aaa88 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -379,7 +379,7 @@ See here the methods that you can override in your custom Files Pipeline: By default the :meth:`get_media_requests` method returns ``None`` which means there are no files to download for the item. - .. method:: FilesPipeline.item_completed(results, items, info) + .. method:: FilesPipeline.item_completed(results, item, info) The :meth:`FilesPipeline.item_completed` method called when all file requests for a single item have completed (either finished downloading, or @@ -422,7 +422,7 @@ See here the methods that you can override in your custom Images Pipeline: Must return a Request for each image URL. - .. method:: ImagesPipeline.item_completed(results, items, info) + .. method:: ImagesPipeline.item_completed(results, item, info) The :meth:`ImagesPipeline.item_completed` method is called when all image requests for a single item have completed (either finished downloading, or diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7dae68470..25ae4b5ba 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -238,7 +238,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_ + services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a + super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is `Crawlera`_ @@ -253,3 +254,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: http://scrapinghub.com/crawlera +.. _scrapoxy: http://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75b98d3b3..664a7239f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -299,6 +299,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`download_timeout` * :reqmeta:`download_maxsize` +* :reqmeta:`download_latency` * :reqmeta:`proxy` .. reqmeta:: bindaddress @@ -316,6 +317,15 @@ download_timeout The amount of time (in secs) that the downloader will wait before timing out. See also: :setting:`DOWNLOAD_TIMEOUT`. +.. reqmeta:: download_latency + +download_latency +---------------- + +The amount of time spent to fetch the response, since the request has been +started, i.e. HTTP message sent over the network. This meta key only becomes +available when the response has been downloaded. While most other meta keys are +used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: @@ -507,7 +517,13 @@ Response objects .. attribute:: Response.headers - A dictionary-like object which contains the response headers. + A dictionary-like object which contains the response headers. Values can + be accessed using :meth:`get` to return the first header value with the + specified name or :meth:`getlist` to return all header values with the + specified name. For example, this call will give you all cookies in the + headers:: + + response.headers.getlist('Set-Cookie') .. attribute:: Response.body diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..0515a9e0d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -468,7 +468,6 @@ Default:: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } @@ -789,6 +788,16 @@ If ``True``, all standard output (and error) of your process will be redirected to the log. For example if you ``print 'hello'`` it will appear in the Scrapy log. +.. setting:: LOG_SHORT_NAMES + +LOG_SHORT_NAMES +--------------- + +Default: ``False`` + +If ``True``, the logs will just contain the root path. If it is set to ``False`` +then it displays the component responsible for the log output + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED @@ -1028,11 +1037,40 @@ Stats counter (``scheduler/unserializable``) tracks the number of times this hap Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + 1956-01-31 00:00:00+0800 [scrapy.core.scheduler] ERROR: Unable to serialize request: - reason: cannot serialize (type Request)> - no more unserializable requests will be logged (see 'scheduler/unserializable' stats counter) + +.. setting:: SCHEDULER_DISK_QUEUE + +SCHEDULER_DISK_QUEUE +-------------------- + +Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` + +Type of disk queue that will be used by scheduler. Other available types are +``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +``scrapy.squeues.MarshalLifoDiskQueue``. + +.. setting:: SCHEDULER_MEMORY_QUEUE + +SCHEDULER_MEMORY_QUEUE +---------------------- +Default: ``'scrapy.squeues.LifoMemoryQueue'`` + +Type of in-memory queue used by scheduler. Other available type is: +``scrapy.squeues.FifoMemoryQueue``. + +.. setting:: SCHEDULER_PRIORITY_QUEUE + +SCHEDULER_PRIORITY_QUEUE +------------------------ +Default: ``'queuelib.PriorityQueue'`` + +Type of priority queue used by scheduler. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..ef6aeeed3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,49 +161,65 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() - u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' + 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() - [u'reddit: the front page of the internet'] + ['reddit: the front page of the internet'] >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses @@ -234,8 +254,8 @@ Here's an example of how you would call it from your spider:: When you run the spider, you will get something similar to this:: - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) - 2014-01-23 17:48:31-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] crawler ... @@ -258,7 +278,7 @@ Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the crawling:: >>> ^D - 2014-01-23 17:50:03-0400 [scrapy] DEBUG: Crawled (200) (referer: None) + 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a38c1ab65..604f1864f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -28,7 +28,12 @@ The :setting:`SPIDER_MIDDLEWARES` setting is merged with the :setting:`SPIDER_MIDDLEWARES_BASE` setting defined in Scrapy (and not meant to be overridden) and then sorted by order to get the final sorted list of enabled middlewares: the first middleware is the one closer to the engine and the last -is the one closer to the spider. +is the one closer to the spider. In other words, +the :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_input` +method of each middleware will be invoked in increasing +middleware order (100, 200, 300, ...), and the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of each middleware will be invoked in decreasing order. To decide which order to assign to your middleware see the :setting:`SPIDER_MIDDLEWARES_BASE` setting and pick a value according to where diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0e473709a..c123c2635 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -24,8 +24,8 @@ For spiders, the scraping cycle goes through something like this: Requests. 2. In the callback function, you parse the response (web page) and return either - dicts with extracted data, :class:`~scrapy.item.Item` objects, - :class:`~scrapy.http.Request` objects, or an iterable of these objects. + dicts with extracted data, :class:`~scrapy.item.Item` objects, + :class:`~scrapy.http.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their response handled by the specified callback. @@ -56,7 +56,7 @@ scrapy.Spider must inherit (including spiders that come bundled with Scrapy, as well as spiders that you write yourself). It doesn't provide any special functionality. It just provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` + the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` for each of the resulting responses. .. attribute:: name @@ -72,6 +72,8 @@ scrapy.Spider spider that crawls ``mywebsite.com`` would often be called ``mywebsite``. + .. note:: In Python 2 this must be ASCII only. + .. attribute:: allowed_domains An optional list of strings containing domains that this spider is @@ -159,7 +161,7 @@ scrapy.Spider class MySpider(scrapy.Spider): name = 'myspider' - + def start_requests(self): return [scrapy.FormRequest("http://www.example.com/login", formdata={'user': 'john', 'pass': 'secret'}, @@ -245,8 +247,8 @@ Return multiple Requests and items from a single callback:: for url in response.xpath('//a/@href').extract(): yield scrapy.Request(url, callback=self.parse) - -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; + +Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; to give data more structure you can use :ref:`topics-items`:: import scrapy @@ -255,7 +257,7 @@ to give data more structure you can use :ref:`topics-items`:: class MySpider(scrapy.Spider): name = 'example.com' allowed_domains = ['example.com'] - + def start_requests(self): yield scrapy.Request('http://www.example.com/1.html', self.parse) yield scrapy.Request('http://www.example.com/2.html', self.parse) @@ -267,7 +269,7 @@ to give data more structure you can use :ref:`topics-items`:: for url in response.xpath('//a/@href').extract(): yield scrapy.Request(url, callback=self.parse) - + .. _spiderargs: Spider arguments @@ -283,7 +285,7 @@ Spider arguments are passed through the :command:`crawl` command using the scrapy crawl myspider -a category=electronics -Spiders receive arguments in their constructors:: +Spiders can access arguments in their `__init__` methods:: import scrapy @@ -295,11 +297,42 @@ Spiders receive arguments in their constructors:: self.start_urls = ['http://www.example.com/categories/%s' % category] # ... +The default `__init__` method will take any spider arguments +and copy them to the spider as attributes. +The above example can also be written as follows:: + + import scrapy + + class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + +Keep in mind that spider arguments are only strings. +The spider will not do any parsing on its own. +If you were to set the `start_urls` attribute from the command line, +you would have to parse it on your own into a list +using something like +`ast.literal_eval `_ +or `json.loads `_ +and then set it as an attribute. +Otherwise, you would cause iteration over a `start_urls` string +(a very common python pitfall) +resulting in each character being seen as a separate url. + +A valid use case is to set the http auth credentials +used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` +or the user agent +used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: + + scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword -a user_agent=mybot + Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. .. _builtin-spiders: - + Generic Spiders =============== diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 102df581c..679bb56ff 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -1,3 +1,5 @@ +:orphan: Ubuntu packages are obsolete + .. _topics-ubuntu: =============== @@ -11,6 +13,9 @@ those in Ubuntu, and more stable too since they're continuously built from `GitHub repo`_ (master & stable branches) and so they contain the latest bug fixes. +.. caution:: These packages are currently not updated and may not work on + Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`. + To use the packages: 1. Import the GPG key used to sign Scrapy packages into APT keyring:: diff --git a/docs/versioning.rst b/docs/versioning.rst index 8e7908762..0421ba544 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -7,22 +7,33 @@ Versioning and API Stability Versioning ========== -Scrapy uses the `odd-numbered versions for development releases`_. - There are 3 numbers in a Scrapy version: *A.B.C* * *A* is the major version. This will rarely change and will signify very large changes. * *B* is the release number. This will include many changes including features - and things that possibly break backwards compatibility. Even Bs will be - stable branches, and odd Bs will be development. + and things that possibly break backwards compatibility, although we strive to + keep theses cases at a minimum. * *C* is the bugfix release number. +Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, +and may require special attention before upgrading. + +Development releases do not follow 3-numbers version and are generally +released as ``dev`` suffixed versions, e.g. ``1.3dev``. + +.. note:: + With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_. + This is not the case anymore from Scrapy 1.0 onwards. + + Starting with Scrapy 1.0, all releases should be considered production-ready. + For example: -* *0.14.1* is the first bugfix release of the *0.14* series (safe to use in +* *1.1.1* is the first bugfix release of the *1.1* series (safe to use in production) + API Stability ============= diff --git a/requirements.txt b/requirements.txt index cfa907050..64b6e771c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=10.0.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/scrapy/VERSION b/scrapy/VERSION index a25fc280b..f0bb29e76 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.2.0dev2 +1.3.0 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..7d4840529 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -27,6 +28,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +53,12 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + # by default, let the framework handle redirects, + # i.e. command handles all codes expect 3xx + if not opts.no_redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6a8978415..5264982b6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -121,8 +121,8 @@ class Command(ScrapyCommand): def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): for rule in spider.rules: - if rule.link_extractor.matches(response.url) and rule.callback: - return rule.callback + if rule.link_extractor.matches(response.url): + return rule.callback or "parse" else: logger.error('No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', @@ -166,6 +166,11 @@ class Command(ScrapyCommand): if not cb: if opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) + + if not cb: + logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', + {'url': response.url, 'spider': spider.name}) + return else: cb = 'parse' diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..40a58d94a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e3989baaf..594106632 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -17,6 +17,7 @@ TEMPLATES_TO_RENDER = ( ('${project_name}', 'settings.py.tmpl'), ('${project_name}', 'items.py.tmpl'), ('${project_name}', 'pipelines.py.tmpl'), + ('${project_name}', 'middlewares.py.tmpl'), ) IGNORE = ignore_patterns('*.pyc', '.svn') diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 4bf085c9e..a9954edb0 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -26,12 +26,25 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: + import cssselect + import parsel import lxml.etree + import w3lib + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + print("Scrapy : %s" % scrapy.__version__) print("lxml : %s" % lxml_version) print("libxml2 : %s" % libxml2_version) + print("cssselect : %s" % cssselect.__version__) + print("parsel : %s" % parsel.__version__) + print("w3lib : %s" % w3lib_version) print("Twisted : %s" % twisted.version.short()) print("Python : %s" % sys.version.replace("\n", "- ")) print("pyOpenSSL : %s" % self._get_openssl_version()) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b643d935b..4bd4c6166 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -13,7 +13,7 @@ try: from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS - from scrapy.core.downloader.tls import ScrapyClientTLSOptions + from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS @implementer(IPolicyForHTTPS) @@ -46,7 +46,9 @@ try: # not calling super(..., self).__init__ return CertificateOptions(verify=False, method=getattr(self, 'method', - getattr(self, '_ssl_method', None))) + getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=DEFAULT_CIPHERS) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 404e9160b..ecd7f90d3 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,8 +13,9 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import PotentialDataLoss -from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, TCP4ClientEndpoint +from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ + HTTPConnectionPool +from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes @@ -318,14 +319,13 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ("Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize})." - ).format(url=request.url, size=expected_size, maxsize=maxsize) + error_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} - logger.error(error_message) + logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_message) + raise defer.CancelledError(error_msg % error_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 00c94ee2e..955b7630c 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -28,6 +28,7 @@ try: SSL_CB_HANDSHAKE_START = 0x10 SSL_CB_HANDSHAKE_DONE = 0x20 + from twisted.internet.ssl import AcceptableCiphers from twisted.internet._sslverify import (ClientTLSOptions, _maybeSetHostNameIndication, verifyHostname, @@ -60,6 +61,8 @@ try: 'from host "{}" (exception: {})'.format( self._hostnameASCII, repr(e))) + DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') + except ImportError: # ImportError should not matter for older Twisted versions # as the above is not used in the fallback ScrapyClientContextFactory diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3c4bc662c..2b5770138 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -48,7 +48,8 @@ class Slot(object): if self.closing and not self.inprogress: if self.nextcall: self.nextcall.cancel() - self.heartbeat.stop() + if self.heartbeat.running: + self.heartbeat.stop() self.closing.callback(None) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index dcd6fb989..a54b4daf0 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -89,8 +89,8 @@ class Scheduler(object): msg = ("Unable to serialize request: %(request)s - reason:" " %(reason)s - no more unserializable requests will be" " logged (stats being collected)") - logger.error(msg, {'request': request, 'reason': e}, - exc_info=True, extra={'spider': self.spider}) + logger.warning(msg, {'request': request, 'reason': e}, + exc_info=True, extra={'spider': self.spider}) self.logunser = False self.stats.inc_value('scheduler/unserializable', spider=self.spider) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 67198179d..c08e37367 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -119,7 +119,7 @@ class Scraper(object): self._scrape(response, request, spider).chainDeferred(deferred) def _scrape(self, response, request, spider): - """Handle the downloaded response or failure trough the spider + """Handle the downloaded response or failure through the spider callback/errback""" assert isinstance(response, (Response, Failure)) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 57e97e4d2..64d94c489 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -1,6 +1,14 @@ +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer +warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " + "chunked transfers are supported by default.", + ScrapyDeprecationWarning, stacklevel=2) + + class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 521327bfe..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import signals from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -from scrapy.xlib.tx import ResponseFailed class HttpCacheMiddleware(object): diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4ed7e4c24..db276eefb 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,9 +1,10 @@ import logging from six.moves.urllib.parse import urljoin +from w3lib.url import safe_url_string + from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh -from scrapy.utils.python import to_native_str from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) @@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if 'Location' not in response.headers or response.status not in allowed_status: return response - # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8 - location = to_native_str(response.headers['location'].decode('latin1')) + location = safe_url_string(response.headers['location']) redirected_url = urljoin(request.url, location) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 74938067f..c9c512be8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -17,10 +17,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message -from scrapy.xlib.tx import ResponseFailed from scrapy.core.downloader.handlers.http11 import TunnelError logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 698f394ad..c3dfa7819 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -13,6 +13,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import to_native_str logger = logging.getLogger(__name__) @@ -40,7 +41,8 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch(self._useragent, request.url): + if rp is not None and not rp.can_fetch( + to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) raise IgnoreRequest() @@ -94,7 +96,9 @@ class RobotsTxtMiddleware(object): # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. pass - rp.parse(body.splitlines()) + # stdlib's robotparser expects native 'str' ; + # with unicode input, non-ASCII encoded bytes decoding fails in Python2 + rp.parse(to_native_str(body).splitlines()) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c3fc66de5..85d328528 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -170,6 +170,7 @@ class FeedExporter(object): if not self._exporter_supported(self.format): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') + self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,14 +189,18 @@ class FeedExporter(object): file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, encoding=self.export_encoding) - exporter.start_exporting() + if self.store_empty: + exporter.start_exporting() + self._exporting = True self.slot = SpiderSlot(file, exporter, storage, uri) def close_spider(self, spider): slot = self.slot if not slot.itemcount and not self.store_empty: return - slot.exporter.finish_exporting() + if self._exporting: + slot.exporter.finish_exporting() + self._exporting = False logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" log_args = {'format': self.format, 'itemcount': slot.itemcount, @@ -210,6 +215,9 @@ class FeedExporter(object): def item_scraped(self, item, spider): slot = self.slot + if not self._exporting: + slot.exporter.start_exporting() + self._exporting = True slot.exporter.export_item(item) slot.itemcount += 1 return item diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 647e50f8d..b685e7b19 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -15,6 +15,7 @@ class LogStats(object): self.stats = stats self.interval = interval self.multiplier = 60.0 / self.interval + self.task = None @classmethod def from_crawler(cls, crawler): @@ -47,5 +48,5 @@ class LogStats(object): logger.info(msg, log_args, extra={'spider': spider}) def spider_closed(self, spider, reason): - if self.task.running: + if self.task and self.task.running: self.task.stop() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 983154001..58ad414f1 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -9,6 +9,8 @@ from six.moves.urllib.parse import urljoin from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref from scrapy.http.common import obsolete_setter +from scrapy.exceptions import NotSupported + class Response(object_ref): @@ -80,3 +82,22 @@ class Response(object_ref): """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" return urljoin(self.url, url) + + @property + def text(self): + """For subclasses of TextResponse, this will return the body + as text (unicode object in Python 2 and str in Python 3) + """ + raise AttributeError("Response content isn't text") + + def css(self, *a, **kw): + """Shortcut method implemented only by responses whose content + is text (subclasses of TextResponse). + """ + raise NotSupported("Response content isn't text") + + def xpath(self, *a, **kw): + """Shortcut method implemented only by responses whose content + is text (subclasses of TextResponse). + """ + raise NotSupported("Response content isn't text") diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index dcc261b31..9867e1179 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -3,6 +3,7 @@ HTMLParser-based link extractor """ import warnings +import six from six.moves.html_parser import HTMLParser from six.moves.urllib.parse import urljoin @@ -39,7 +40,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, unicode): + if isinstance(link.url, six.text_type): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 9938e071f..c68dae4c8 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,6 +1,7 @@ """ SGMLParser-based Link extractors """ +import six from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser @@ -40,7 +41,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, unicode): + if isinstance(link.url, six.text_type): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/mail.py b/scrapy/mail.py index c6339f25b..0bb395521 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -21,6 +21,8 @@ else: from twisted.internet import defer, reactor, ssl +from .utils.misc import arg_to_iter + logger = logging.getLogger(__name__) @@ -48,6 +50,10 @@ class MailSender(object): msg = MIMEMultipart() else: msg = MIMENonMultipart(*mimetype.split('/', 1)) + + to = list(arg_to_iter(to)) + cc = list(arg_to_iter(cc)) + msg['From'] = self.mailfrom msg['To'] = COMMASPACE.join(to) msg['Date'] = formatdate(localtime=True) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8cdc548f6..843b4d3ec 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -233,7 +233,8 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, - base_class_name=cls_name) + base_class_name=cls_name, + settings=settings) self.expires = settings.getint( resolve('FILES_EXPIRES'), self.EXPIRES ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index af5825c0b..5796bfb80 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -55,7 +55,8 @@ class ImagesPipeline(FilesPipeline): settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="ImagesPipeline") + base_class_name="ImagesPipeline", + settings=settings) self.expires = settings.getint( resolve("IMAGES_EXPIRES"), self.EXPIRES ) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 82b4b462e..57f70499e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,7 +28,8 @@ class MediaPipeline(object): self.download_func = download_func - def _key_for_pipe(self, key, base_class_name=None): + def _key_for_pipe(self, key, base_class_name=None, + settings=None): """ >>> MediaPipeline()._key_for_pipe("IMAGES") 'IMAGES' @@ -38,9 +39,11 @@ class MediaPipeline(object): 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - if class_name == base_class_name or not base_class_name: + formatted_key = "{}_{}".format(class_name.upper(), key) + if class_name == base_class_name or not base_class_name \ + or (settings and not settings.get(formatted_key)): return key - return "{}_{}".format(class_name.upper(), key) + return formatted_key @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..24714a7a8 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side @@ -192,6 +191,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/shell.py b/scrapy/shell.py index 099e1af0a..6f94635a1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -20,6 +20,7 @@ from scrapy.item import BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.conf import get_config @@ -40,11 +41,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, redirect=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, redirect=redirect) elif request: self.fetch(request, spider) elif response: @@ -98,14 +99,16 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread( @@ -115,6 +118,9 @@ class Shell(object): self.populate_vars(response, request, spider) def populate_vars(self, response=None, request=None, spider=None): + import scrapy + + self.vars['scrapy'] = scrapy self.vars['crawler'] = self.crawler self.vars['item'] = self.item_class() self.vars['settings'] = self.crawler.settings @@ -136,14 +142,18 @@ class Shell(object): def get_help(self): b = [] b.append("Available Scrapy objects:") + b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index fbf68cec4..d4f0f663f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import traceback +import warnings from zope.interface import implementer @@ -18,15 +20,21 @@ class SpiderLoader(object): self.spider_modules = settings.getlist('SPIDER_MODULES') self._spiders = {} self._load_all_spiders() - + def _load_spiders(self, module): for spcls in iter_spider_classes(module): self._spiders[spcls.name] = spcls def _load_all_spiders(self): for name in self.spider_modules: - for module in walk_modules(name): - self._load_spiders(module) + try: + for module in walk_modules(name): + self._load_spiders(module) + except ImportError as e: + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "Check SPIDER_MODULES setting".format( + modname=name, tb=traceback.format_exc())) + warnings.warn(msg, RuntimeWarning) @classmethod def from_settings(cls, settings): diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 6b7c50fce..e34c265c1 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -46,7 +46,7 @@ class HttpErrorMiddleware(object): def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): - logger.debug( + logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {'response': response}, extra={'spider': spider}, ) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 89d96c330..9e45637c3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -32,7 +32,7 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): - for url in sitemap_urls_from_robots(response.text): + for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) else: body = self._get_sitemap_body(response) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl new file mode 100644 index 000000000..42318fec2 --- /dev/null +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your spider middleware +# +# See documentation in: +# http://doc.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + + +class ${ProjectName}SpiderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, dict or Item objects. + for i in result: + yield i + + def process_spider_exception(response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Response, dict + # or Item objects. + pass + + def process_start_requests(start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index f13e85871..72f25ebef 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -47,7 +47,7 @@ ROBOTSTXT_OBEY = True # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomSpiderMiddleware': 543, +# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2d3b96e38..99e5d43b2 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -5,9 +5,7 @@ import scrapy class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] - start_urls = ( - 'http://www.$domain/', - ) + start_urls = ['http://$domain/'] def parse(self, response): pass diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 154237d9c..802cb88a1 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/'] + start_urls = ['http://$domain/'] rules = ( Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 0544e0ae7..c2e4bacfe 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/feed.csv'] + start_urls = ['http://$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index d8ff61f6e..7c2ff8850 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/feed.xml'] + start_urls = ['http://$domain/feed.xml'] iterator = 'iternodes' # you can change this; see the docs itertag = 'item' # change it accordingly diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a712df30e..1888d9599 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -13,7 +13,12 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - shell = InteractiveShellEmbed( + # Always use .instace() to ensure _instance propagation to all parents + # this is needed for completion works well for new imports + # and clear the instance to always have the fresh env + # on repeated breaks like with inspect_response() + InteractiveShellEmbed.clear_instance() + shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config) shell() return wrapper diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d04b43176..e516185bd 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -304,3 +304,13 @@ class LocalCache(OrderedDict): while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) + + +class SequenceExclude(object): + """Object to test if an item is NOT within some sequence.""" + + def __init__(self, seq): + self.seq = seq + + def __contains__(self, item): + return item not in self.seq diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index d3fde097c..0ea91872b 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -131,7 +131,8 @@ def _get_handler(settings): ) handler.setFormatter(formatter) handler.setLevel(settings.get('LOG_LEVEL')) - handler.addFilter(TopLevelFormatter(['scrapy'])) + if settings.getbool('LOG_SHORT_NAMES'): + handler.addFilter(TopLevelFormatter(['scrapy'])) return handler @@ -158,6 +159,10 @@ class StreamLogger(object): for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) + def flush(self): + for h in self.logger.handlers: + h.flush() + class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a15a0d90f..95c6a8035 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -12,6 +12,7 @@ from scrapy.exceptions import NotConfigured ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' + def inside_project(): scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') if scrapy_module is not None: @@ -23,6 +24,7 @@ def inside_project(): return True return bool(closest_scrapy_cfg()) + def project_data_dir(project='default'): """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): @@ -39,16 +41,22 @@ def project_data_dir(project='default'): os.makedirs(d) return d + def data_path(path, createdir=False): - """If path is relative, return the given path inside the project data dir, - otherwise return the path unmodified + """ + Return the given path joined with the .scrapy data directory. + If given an absolute path, return it unmodified. """ if not isabs(path): - path = join(project_data_dir(), path) + if inside_project(): + path = join(project_data_dir(), path) + else: + path = join('.scrapy', path) if createdir and not exists(path): os.makedirs(path) return path + def get_project_settings(): if ENVVAR not in os.environ: project = os.environ.get('SCRAPY_PROJECT', 'default') diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 008196435..4742b3e13 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,9 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + import lxml.etree +from six.moves.urllib.parse import urljoin class Sitemap(object): @@ -34,10 +36,11 @@ class Sitemap(object): yield d -def sitemap_urls_from_robots(robots_text): +def sitemap_urls_from_robots(robots_text, base_url=None): """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith('sitemap:'): - yield line.split(':', 1)[1].strip() + url = line.split(':', 1)[1].strip() + yield urljoin(base_url, url) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ad0375443..e50a989b3 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -20,12 +20,20 @@ class SiteTest(object): return urljoin(self.baseurl, path) +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request): + content = util.Redirect.render(self, request) + return content.replace(b'http-equiv=\"refresh\"', + b'http-no-equiv=\"do-not-refresh-me\"') + + def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) r.putChild(b"html", static.Data(b"

      Works

      World

      ", "text/html")) r.putChild(b"enc-gb18030", static.Data(b"

      gb18030 encoding

      ", "text/html; charset=gb18030")) r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) diff --git a/scrapy/xlib/tx.py b/scrapy/xlib/tx.py new file mode 100644 index 000000000..0d94307b7 --- /dev/null +++ b/scrapy/xlib/tx.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning + +from twisted.web import client +from twisted.internet import endpoints + +Agent = client.Agent # since < 11.1 +ProxyAgent = client.ProxyAgent # since 11.1 +ResponseDone = client.ResponseDone # since 11.1 +ResponseFailed = client.ResponseFailed # since 11.1 +HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 +TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 + +warnings.warn("Importing from scrapy.xlib.tx is deprecated and will" + " no longer be supported in future Scrapy versions." + " Update your code to import from twisted proper.", + ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/xlib/tx/LICENSE b/scrapy/xlib/tx/LICENSE deleted file mode 100644 index 8529f6edf..000000000 --- a/scrapy/xlib/tx/LICENSE +++ /dev/null @@ -1,57 +0,0 @@ -Copyright (c) 2001-2013 -Allen Short -Andy Gayton -Andrew Bennetts -Antoine Pitrou -Apple Computer, Inc. -Benjamin Bruheim -Bob Ippolito -Canonical Limited -Christopher Armstrong -David Reid -Donovan Preston -Eric Mangold -Eyal Lotem -Itamar Turner-Trauring -James Knight -Jason A. Mobarak -Jean-Paul Calderone -Jessica McKellar -Jonathan Jacobs -Jonathan Lange -Jonathan D. Simms -Jürgen Hermann -Kevin Horn -Kevin Turner -Mary Gardiner -Matthew Lefkowitz -Massachusetts Institute of Technology -Moshe Zadka -Paul Swartz -Pavel Pergamenshchik -Ralph Meijer -Sean Riley -Software Freedom Conservancy -Travis B. Hartwell -Thijs Triemstra -Thomas Herve -Timothy Allen - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scrapy/xlib/tx/README b/scrapy/xlib/tx/README deleted file mode 100644 index 75ef485ce..000000000 --- a/scrapy/xlib/tx/README +++ /dev/null @@ -1,2 +0,0 @@ -This source files are adapted copies from Twisted trunk to support HTTP1.1 -handler under Twisted >= 11.1 and Twisted <= 13.0.0 diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py deleted file mode 100644 index 1ac4e0108..000000000 --- a/scrapy/xlib/tx/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -from scrapy import twisted_version -if twisted_version > (13, 0, 0): - from twisted.web import client - from twisted.internet import endpoints -if twisted_version >= (11, 1, 0): - from . import client, endpoints -else: - from scrapy.exceptions import NotSupported - class _Mocked(object): - def __init__(self, *args, **kw): - raise NotSupported('HTTP1.1 not supported') - class _Mock(object): - def __getattr__(self, name): - return _Mocked - client = endpoints = _Mock() - - -Agent = client.Agent -ProxyAgent = client.ProxyAgent -ResponseDone = client.ResponseDone -ResponseFailed = client.ResponseFailed -HTTPConnectionPool = client.HTTPConnectionPool -TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py deleted file mode 100644 index 16d0ca6b4..000000000 --- a/scrapy/xlib/tx/_newclient.py +++ /dev/null @@ -1,1516 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_newclient -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -An U{HTTP 1.1} client. - -The way to use the functionality provided by this module is to: - - - Connect a L{HTTP11ClientProtocol} to an HTTP server - - Create a L{Request} with the appropriate data - - Pass the request to L{HTTP11ClientProtocol.request} - - The returned Deferred will fire with a L{Response} object - - Create a L{IProtocol} provider which can handle the response body - - Connect it to the response with L{Response.deliverBody} - - When the protocol's C{connectionLost} method is called, the response is - complete. See L{Response.deliverBody} for details. - -Various other classes in this module support this usage: - - - HTTPParser is the basic HTTP parser. It can handle the parts of HTTP which - are symmetric between requests and responses. - - - HTTPClientParser extends HTTPParser to handle response-specific parts of - HTTP. One instance is created for each request to parse the corresponding - response. -""" - -__metaclass__ = type - -from zope.interface import implements - -from twisted.python import log -from twisted.python.reflect import fullyQualifiedName -from twisted.python.failure import Failure -from twisted.internet.interfaces import IConsumer, IPushProducer -from twisted.internet.error import ConnectionDone -from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred -from twisted.internet.defer import CancelledError -from twisted.internet.protocol import Protocol -from twisted.protocols.basic import LineReceiver -from twisted.web.http_headers import Headers -from twisted.web.http import NO_CONTENT, NOT_MODIFIED -from twisted.web.http import _DataLoss, PotentialDataLoss -from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder - -from .iweb import IResponse, UNKNOWN_LENGTH - -# States HTTPParser can be in -STATUS = 'STATUS' -HEADER = 'HEADER' -BODY = 'BODY' -DONE = 'DONE' - - -class BadHeaders(Exception): - """ - Headers passed to L{Request} were in some way invalid. - """ - - - -class ExcessWrite(Exception): - """ - The body L{IBodyProducer} for a request tried to write data after - indicating it had finished writing data. - """ - - -class ParseError(Exception): - """ - Some received data could not be parsed. - - @ivar data: The string which could not be parsed. - """ - def __init__(self, reason, data): - Exception.__init__(self, reason, data) - self.data = data - - - -class BadResponseVersion(ParseError): - """ - The version string in a status line was unparsable. - """ - - - -class _WrapperException(Exception): - """ - L{_WrapperException} is the base exception type for exceptions which - include one or more other exceptions as the low-level causes. - - @ivar reasons: A list of exceptions. See subclass documentation for more - details. - """ - def __init__(self, reasons): - Exception.__init__(self, reasons) - self.reasons = reasons - - - -class RequestGenerationFailed(_WrapperException): - """ - There was an error while creating the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request generation was considered to have failed. - """ - - - -class RequestTransmissionFailed(_WrapperException): - """ - There was an error while sending the bytes which make up a request. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the request transmission was considered to have failed. - """ - - - -class ConnectionAborted(Exception): - """ - The connection was explicitly aborted by application code. - """ - - - -class WrongBodyLength(Exception): - """ - An L{IBodyProducer} declared the number of bytes it was going to - produce (via its C{length} attribute) and then produced a different number - of bytes. - """ - - - -class ResponseDone(Exception): - """ - L{ResponseDone} may be passed to L{IProtocol.connectionLost} on the - protocol passed to L{Response.deliverBody} and indicates that the entire - response has been delivered. - """ - - - -class ResponseFailed(_WrapperException): - """ - L{ResponseFailed} indicates that all of the response to a request was not - received for some reason. - - @ivar reasons: A C{list} of one or more L{Failure} instances giving the - reasons the response was considered to have failed. - - @ivar response: If specified, the L{Response} received from the server (and - in particular the status code and the headers). - """ - - def __init__(self, reasons, response=None): - _WrapperException.__init__(self, reasons) - self.response = response - - - -class ResponseNeverReceived(ResponseFailed): - """ - A L{ResponseFailed} that knows no response bytes at all have been received. - """ - - - -class RequestNotSent(Exception): - """ - L{RequestNotSent} indicates that an attempt was made to issue a request but - for reasons unrelated to the details of the request itself, the request - could not be sent. For example, this may indicate that an attempt was made - to send a request using a protocol which is no longer connected to a - server. - """ - - - -def _callAppFunction(function): - """ - Call C{function}. If it raises an exception, log it with a minimal - description of the source. - - @return: C{None} - """ - try: - function() - except: - log.err(None, "Unexpected exception from %s" % ( - fullyQualifiedName(function),)) - - - -class HTTPParser(LineReceiver): - """ - L{HTTPParser} handles the parsing side of HTTP processing. With a suitable - subclass, it can parse either the client side or the server side of the - connection. - - @ivar headers: All of the non-connection control message headers yet - received. - - @ivar state: State indicator for the response parsing state machine. One - of C{STATUS}, C{HEADER}, C{BODY}, C{DONE}. - - @ivar _partialHeader: C{None} or a C{list} of the lines of a multiline - header while that header is being received. - """ - - # NOTE: According to HTTP spec, we're supposed to eat the - # 'Proxy-Authenticate' and 'Proxy-Authorization' headers also, but that - # doesn't sound like a good idea to me, because it makes it impossible to - # have a non-authenticating transparent proxy in front of an authenticating - # proxy. An authenticating proxy can eat them itself. -jknight - # - # Further, quoting - # http://homepages.tesco.net/J.deBoynePollard/FGA/web-proxy-connection-header.html - # regarding the 'Proxy-Connection' header: - # - # The Proxy-Connection: header is a mistake in how some web browsers - # use HTTP. Its name is the result of a false analogy. It is not a - # standard part of the protocol. There is a different standard - # protocol mechanism for doing what it does. And its existence - # imposes a requirement upon HTTP servers such that no proxy HTTP - # server can be standards-conforming in practice. - # - # -exarkun - - # Some servers (like http://news.ycombinator.com/) return status lines and - # HTTP headers delimited by \n instead of \r\n. - delimiter = '\n' - - CONNECTION_CONTROL_HEADERS = set([ - 'content-length', 'connection', 'keep-alive', 'te', 'trailers', - 'transfer-encoding', 'upgrade', 'proxy-connection']) - - def connectionMade(self): - self.headers = Headers() - self.connHeaders = Headers() - self.state = STATUS - self._partialHeader = None - - - def switchToBodyMode(self, decoder): - """ - Switch to body parsing mode - interpret any more bytes delivered as - part of the message body and deliver them to the given decoder. - """ - if self.state == BODY: - raise RuntimeError("already in body mode") - - self.bodyDecoder = decoder - self.state = BODY - self.setRawMode() - - - def lineReceived(self, line): - """ - Handle one line from a response. - """ - # Handle the normal CR LF case. - if line[-1:] == '\r': - line = line[:-1] - - if self.state == STATUS: - self.statusReceived(line) - self.state = HEADER - elif self.state == HEADER: - if not line or line[0] not in ' \t': - if self._partialHeader is not None: - header = ''.join(self._partialHeader) - name, value = header.split(':', 1) - value = value.strip() - self.headerReceived(name, value) - if not line: - # Empty line means the header section is over. - self.allHeadersReceived() - else: - # Line not beginning with LWS is another header. - self._partialHeader = [line] - else: - # A line beginning with LWS is a continuation of a header - # begun on a previous line. - self._partialHeader.append(line) - - - def rawDataReceived(self, data): - """ - Pass data from the message body to the body decoder object. - """ - self.bodyDecoder.dataReceived(data) - - - def isConnectionControlHeader(self, name): - """ - Return C{True} if the given lower-cased name is the name of a - connection control header (rather than an entity header). - - According to RFC 2616, section 14.10, the tokens in the Connection - header are probably relevant here. However, I am not sure what the - practical consequences of either implementing or ignoring that are. - So I leave it unimplemented for the time being. - """ - return name in self.CONNECTION_CONTROL_HEADERS - - - def statusReceived(self, status): - """ - Callback invoked whenever the first line of a new message is received. - Override this. - - @param status: The first line of an HTTP request or response message - without trailing I{CR LF}. - @type status: C{str} - """ - - - def headerReceived(self, name, value): - """ - Store the given header in C{self.headers}. - """ - name = name.lower() - if self.isConnectionControlHeader(name): - headers = self.connHeaders - else: - headers = self.headers - headers.addRawHeader(name, value) - - - def allHeadersReceived(self): - """ - Callback invoked after the last header is passed to C{headerReceived}. - Override this to change to the C{BODY} or C{DONE} state. - """ - self.switchToBodyMode(None) - - - -class HTTPClientParser(HTTPParser): - """ - An HTTP parser which only handles HTTP responses. - - @ivar request: The request with which the expected response is associated. - @type request: L{Request} - - @ivar NO_BODY_CODES: A C{set} of response codes which B{MUST NOT} have a - body. - - @ivar finisher: A callable to invoke when this response is fully parsed. - - @ivar _responseDeferred: A L{Deferred} which will be called back with the - response when all headers in the response have been received. - Thereafter, C{None}. - - @ivar _everReceivedData: C{True} if any bytes have been received. - """ - NO_BODY_CODES = set([NO_CONTENT, NOT_MODIFIED]) - - _transferDecoders = { - 'chunked': _ChunkedTransferDecoder, - } - - bodyDecoder = None - - def __init__(self, request, finisher): - self.request = request - self.finisher = finisher - self._responseDeferred = Deferred() - self._everReceivedData = False - - - def dataReceived(self, data): - """ - Override so that we know if any response has been received. - """ - self._everReceivedData = True - HTTPParser.dataReceived(self, data) - - - def parseVersion(self, strversion): - """ - Parse version strings of the form Protocol '/' Major '.' Minor. E.g. - 'HTTP/1.1'. Returns (protocol, major, minor). Will raise ValueError - on bad syntax. - """ - try: - proto, strnumber = strversion.split('/') - major, minor = strnumber.split('.') - major, minor = int(major), int(minor) - except ValueError as e: - raise BadResponseVersion(str(e), strversion) - if major < 0 or minor < 0: - raise BadResponseVersion("version may not be negative", strversion) - return (proto, major, minor) - - - def statusReceived(self, status): - """ - Parse the status line into its components and create a response object - to keep track of this response's state. - """ - parts = status.split(' ', 2) - if len(parts) != 3: - raise ParseError("wrong number of parts", status) - - try: - statusCode = int(parts[1]) - except ValueError: - raise ParseError("non-integer status code", status) - - self.response = Response( - self.parseVersion(parts[0]), - statusCode, - parts[2], - self.headers, - self.transport) - - - def _finished(self, rest): - """ - Called to indicate that an entire response has been received. No more - bytes will be interpreted by this L{HTTPClientParser}. Extra bytes are - passed up and the state of this L{HTTPClientParser} is set to I{DONE}. - - @param rest: A C{str} giving any extra bytes delivered to this - L{HTTPClientParser} which are not part of the response being - parsed. - """ - self.state = DONE - self.finisher(rest) - - - def isConnectionControlHeader(self, name): - """ - Content-Length in the response to a HEAD request is an entity header, - not a connection control header. - """ - if self.request.method == 'HEAD' and name == 'content-length': - return False - return HTTPParser.isConnectionControlHeader(self, name) - - - def allHeadersReceived(self): - """ - Figure out how long the response body is going to be by examining - headers and stuff. - """ - if (self.response.code in self.NO_BODY_CODES - or self.request.method == 'HEAD'): - self.response.length = 0 - self._finished(self.clearLineBuffer()) - else: - transferEncodingHeaders = self.connHeaders.getRawHeaders( - 'transfer-encoding') - if transferEncodingHeaders: - - # This could be a KeyError. However, that would mean we do not - # know how to decode the response body, so failing the request - # is as good a behavior as any. Perhaps someday we will want - # to normalize/document/test this specifically, but failing - # seems fine to me for now. - transferDecoder = self._transferDecoders[transferEncodingHeaders[0].lower()] - - # If anyone ever invents a transfer encoding other than - # chunked (yea right), and that transfer encoding can predict - # the length of the response body, it might be sensible to - # allow the transfer decoder to set the response object's - # length attribute. - else: - contentLengthHeaders = self.connHeaders.getRawHeaders('content-length') - if contentLengthHeaders is None: - contentLength = None - elif len(contentLengthHeaders) == 1: - contentLength = int(contentLengthHeaders[0]) - self.response.length = contentLength - else: - # "HTTP Message Splitting" or "HTTP Response Smuggling" - # potentially happening. Or it's just a buggy server. - raise ValueError( - "Too many Content-Length headers; response is invalid") - - if contentLength == 0: - self._finished(self.clearLineBuffer()) - transferDecoder = None - else: - transferDecoder = lambda x, y: _IdentityTransferDecoder( - contentLength, x, y) - - if transferDecoder is None: - self.response._bodyDataFinished() - else: - # Make sure as little data as possible from the response body - # gets delivered to the response object until the response - # object actually indicates it is ready to handle bytes - # (probably because an application gave it a way to interpret - # them). - self.transport.pauseProducing() - self.switchToBodyMode(transferDecoder( - self.response._bodyDataReceived, - self._finished)) - - # This must be last. If it were first, then application code might - # change some state (for example, registering a protocol to receive the - # response body). Then the pauseProducing above would be wrong since - # the response is ready for bytes and nothing else would ever resume - # the transport. - self._responseDeferred.callback(self.response) - del self._responseDeferred - - - def connectionLost(self, reason): - if self.bodyDecoder is not None: - try: - try: - self.bodyDecoder.noMoreData() - except PotentialDataLoss: - self.response._bodyDataFinished(Failure()) - except _DataLoss: - self.response._bodyDataFinished( - Failure(ResponseFailed([reason, Failure()], - self.response))) - else: - self.response._bodyDataFinished() - except: - # Handle exceptions from both the except suites and the else - # suite. Those functions really shouldn't raise exceptions, - # but maybe there's some buggy application code somewhere - # making things difficult. - log.err() - elif self.state != DONE: - if self._everReceivedData: - exceptionClass = ResponseFailed - else: - exceptionClass = ResponseNeverReceived - self._responseDeferred.errback(Failure(exceptionClass([reason]))) - del self._responseDeferred - - - -class Request: - """ - A L{Request} instance describes an HTTP request to be sent to an HTTP - server. - - @ivar method: The HTTP method to for this request, ex: 'GET', 'HEAD', - 'POST', etc. - @type method: C{str} - - @ivar uri: The relative URI of the resource to request. For example, - C{'/foo/bar?baz=quux'}. - @type uri: C{str} - - @ivar headers: Headers to be sent to the server. It is important to - note that this object does not create any implicit headers. So it - is up to the HTTP Client to add required headers such as 'Host'. - @type headers: L{twisted.web.http_headers.Headers} - - @ivar bodyProducer: C{None} or an L{IBodyProducer} provider which - produces the content body to send to the remote HTTP server. - - @ivar persistent: Set to C{True} when you use HTTP persistent connection. - @type persistent: C{bool} - """ - def __init__(self, method, uri, headers, bodyProducer, persistent=False): - self.method = method - self.uri = uri - self.headers = headers - self.bodyProducer = bodyProducer - self.persistent = persistent - - - def _writeHeaders(self, transport, TEorCL): - hosts = self.headers.getRawHeaders('host', ()) - if len(hosts) != 1: - raise BadHeaders("Exactly one Host header required") - - # In the future, having the protocol version be a parameter to this - # method would probably be good. It would be nice if this method - # weren't limited to issuing HTTP/1.1 requests. - requestLines = [] - requestLines.append( - '%s %s HTTP/1.1\r\n' % (self.method, self.uri)) - if not self.persistent: - requestLines.append('Connection: close\r\n') - if TEorCL is not None: - requestLines.append(TEorCL) - for name, values in self.headers.getAllRawHeaders(): - requestLines.extend(['%s: %s\r\n' % (name, v) for v in values]) - requestLines.append('\r\n') - transport.writeSequence(requestLines) - - - def _writeToChunked(self, transport): - """ - Write this request to the given transport using chunked - transfer-encoding to frame the body. - """ - self._writeHeaders(transport, 'Transfer-Encoding: chunked\r\n') - encoder = ChunkedEncoder(transport) - encoder.registerProducer(self.bodyProducer, True) - d = self.bodyProducer.startProducing(encoder) - - def cbProduced(ignored): - encoder.unregisterProducer() - def ebProduced(err): - encoder._allowNoMoreWrites() - # Don't call the encoder's unregisterProducer because it will write - # a zero-length chunk. This would indicate to the server that the - # request body is complete. There was an error, though, so we - # don't want to do that. - transport.unregisterProducer() - return err - d.addCallbacks(cbProduced, ebProduced) - return d - - - def _writeToContentLength(self, transport): - """ - Write this request to the given transport using content-length to frame - the body. - """ - self._writeHeaders( - transport, - 'Content-Length: %d\r\n' % (self.bodyProducer.length,)) - - # This Deferred is used to signal an error in the data written to the - # encoder below. It can only errback and it will only do so before too - # many bytes have been written to the encoder and before the producer - # Deferred fires. - finishedConsuming = Deferred() - - # This makes sure the producer writes the correct number of bytes for - # the request body. - encoder = LengthEnforcingConsumer( - self.bodyProducer, transport, finishedConsuming) - - transport.registerProducer(self.bodyProducer, True) - - finishedProducing = self.bodyProducer.startProducing(encoder) - - def combine(consuming, producing): - # This Deferred is returned and will be fired when the first of - # consuming or producing fires. If it's cancelled, forward that - # cancellation to the producer. - def cancelConsuming(ign): - finishedProducing.cancel() - ultimate = Deferred(cancelConsuming) - - # Keep track of what has happened so far. This initially - # contains None, then an integer uniquely identifying what - # sequence of events happened. See the callbacks and errbacks - # defined below for the meaning of each value. - state = [None] - - def ebConsuming(err): - if state == [None]: - # The consuming Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. The - # producing Deferred should not fire later (because the - # consumer should have called stopProducing on the - # producer), but if it does, a callback will be ignored - # and an errback will be logged. - state[0] = 1 - ultimate.errback(err) - else: - # The consuming Deferred errbacked after the producing - # Deferred fired. This really shouldn't ever happen. - # If it does, I goofed. Log the error anyway, just so - # there's a chance someone might notice and complain. - log.err( - err, - "Buggy state machine in %r/[%d]: " - "ebConsuming called" % (self, state[0])) - - def cbProducing(result): - if state == [None]: - # The producing Deferred succeeded first. Nothing will - # ever happen to the consuming Deferred. Tell the - # encoder we're done so it can check what the producer - # wrote and make sure it was right. - state[0] = 2 - try: - encoder._noMoreWritesExpected() - except: - # Fail the overall writeTo Deferred - something the - # producer did was wrong. - ultimate.errback() - else: - # Success - succeed the overall writeTo Deferred. - ultimate.callback(None) - # Otherwise, the consuming Deferred already errbacked. The - # producing Deferred wasn't supposed to fire, but it did - # anyway. It's buggy, but there's not really anything to be - # done about it. Just ignore this result. - - def ebProducing(err): - if state == [None]: - # The producing Deferred failed first. This means the - # overall writeTo Deferred is going to errback now. - # Tell the encoder that we're done so it knows to reject - # further writes from the producer (which should not - # happen, but the producer may be buggy). - state[0] = 3 - encoder._allowNoMoreWrites() - ultimate.errback(err) - else: - # The producing Deferred failed after the consuming - # Deferred failed. It shouldn't have, so it's buggy. - # Log the exception in case anyone who can fix the code - # is watching. - log.err(err, "Producer is buggy") - - consuming.addErrback(ebConsuming) - producing.addCallbacks(cbProducing, ebProducing) - - return ultimate - - d = combine(finishedConsuming, finishedProducing) - def f(passthrough): - # Regardless of what happens with the overall Deferred, once it - # fires, the producer registered way up above the definition of - # combine should be unregistered. - transport.unregisterProducer() - return passthrough - d.addBoth(f) - return d - - - def writeTo(self, transport): - """ - Format this L{Request} as an HTTP/1.1 request and write it to the given - transport. If bodyProducer is not None, it will be associated with an - L{IConsumer}. - - @return: A L{Deferred} which fires with C{None} when the request has - been completely written to the transport or with a L{Failure} if - there is any problem generating the request bytes. - """ - if self.bodyProducer is not None: - if self.bodyProducer.length is UNKNOWN_LENGTH: - return self._writeToChunked(transport) - else: - return self._writeToContentLength(transport) - else: - self._writeHeaders(transport, None) - return succeed(None) - - - def stopWriting(self): - """ - Stop writing this request to the transport. This can only be called - after C{writeTo} and before the L{Deferred} returned by C{writeTo} - fires. It should cancel any asynchronous task started by C{writeTo}. - The L{Deferred} returned by C{writeTo} need not be fired if this method - is called. - """ - # If bodyProducer is None, then the Deferred returned by writeTo has - # fired already and this method cannot be called. - _callAppFunction(self.bodyProducer.stopProducing) - - - -class LengthEnforcingConsumer: - """ - An L{IConsumer} proxy which enforces an exact length requirement on the - total data written to it. - - @ivar _length: The number of bytes remaining to be written. - - @ivar _producer: The L{IBodyProducer} which is writing to this - consumer. - - @ivar _consumer: The consumer to which at most C{_length} bytes will be - forwarded. - - @ivar _finished: A L{Deferred} which will be fired with a L{Failure} if too - many bytes are written to this consumer. - """ - def __init__(self, producer, consumer, finished): - self._length = producer.length - self._producer = producer - self._consumer = consumer - self._finished = finished - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self._finished = None - - - def write(self, bytes): - """ - Write C{bytes} to the underlying consumer unless - C{_noMoreWritesExpected} has been called or there are/have been too - many bytes. - """ - if self._finished is None: - # No writes are supposed to happen any more. Try to convince the - # calling code to stop calling this method by calling its - # stopProducing method and then throwing an exception at it. This - # exception isn't documented as part of the API because you're - # never supposed to expect it: only buggy code will ever receive - # it. - self._producer.stopProducing() - raise ExcessWrite() - - if len(bytes) <= self._length: - self._length -= len(bytes) - self._consumer.write(bytes) - else: - # No synchronous exception is raised in *this* error path because - # we still have _finished which we can use to report the error to a - # better place than the direct caller of this method (some - # arbitrary application code). - _callAppFunction(self._producer.stopProducing) - self._finished.errback(WrongBodyLength("too many bytes written")) - self._allowNoMoreWrites() - - - def _noMoreWritesExpected(self): - """ - Called to indicate no more bytes will be written to this consumer. - Check to see that the correct number have been written. - - @raise WrongBodyLength: If not enough bytes have been written. - """ - if self._finished is not None: - self._allowNoMoreWrites() - if self._length: - raise WrongBodyLength("too few bytes written") - - - -def makeStatefulDispatcher(name, template): - """ - Given a I{dispatch} name and a function, return a function which can be - used as a method and which, when called, will call another method defined - on the instance and return the result. The other method which is called is - determined by the value of the C{_state} attribute of the instance. - - @param name: A string which is used to construct the name of the subsidiary - method to invoke. The subsidiary method is named like C{'_%s_%s' % - (name, _state)}. - - @param template: A function object which is used to give the returned - function a docstring. - - @return: The dispatcher function. - """ - def dispatcher(self, *args, **kwargs): - func = getattr(self, '_' + name + '_' + self._state, None) - if func is None: - raise RuntimeError( - "%r has no %s method in state %s" % (self, name, self._state)) - return func(*args, **kwargs) - dispatcher.__doc__ = template.__doc__ - return dispatcher - - - -class Response: - """ - A L{Response} instance describes an HTTP response received from an HTTP - server. - - L{Response} should not be subclassed or instantiated. - - @ivar _transport: The transport which is delivering this response. - - @ivar _bodyProtocol: The L{IProtocol} provider to which the body is - delivered. C{None} before one has been registered with - C{deliverBody}. - - @ivar _bodyBuffer: A C{list} of the strings passed to C{bodyDataReceived} - before C{deliverBody} is called. C{None} afterwards. - - @ivar _state: Indicates what state this L{Response} instance is in, - particularly with respect to delivering bytes from the response body - to an application-suppled protocol object. This may be one of - C{'INITIAL'}, C{'CONNECTED'}, C{'DEFERRED_CLOSE'}, or C{'FINISHED'}, - with the following meanings: - - - INITIAL: This is the state L{Response} objects start in. No - protocol has yet been provided and the underlying transport may - still have bytes to deliver to it. - - - DEFERRED_CLOSE: If the underlying transport indicates all bytes - have been delivered but no application-provided protocol is yet - available, the L{Response} moves to this state. Data is - buffered and waiting for a protocol to be delivered to. - - - CONNECTED: If a protocol is provided when the state is INITIAL, - the L{Response} moves to this state. Any buffered data is - delivered and any data which arrives from the transport - subsequently is given directly to the protocol. - - - FINISHED: If a protocol is provided in the DEFERRED_CLOSE state, - the L{Response} moves to this state after delivering all - buffered data to the protocol. Otherwise, if the L{Response} is - in the CONNECTED state, if the transport indicates there is no - more data, the L{Response} moves to this state. Nothing else - can happen once the L{Response} is in this state. - """ - implements(IResponse) - - length = UNKNOWN_LENGTH - - _bodyProtocol = None - _bodyFinished = False - - def __init__(self, version, code, phrase, headers, _transport): - self.version = version - self.code = code - self.phrase = phrase - self.headers = headers - self._transport = _transport - self._bodyBuffer = [] - self._state = 'INITIAL' - - - def deliverBody(self, protocol): - """ - Dispatch the given L{IProtocol} depending of the current state of the - response. - """ - deliverBody = makeStatefulDispatcher('deliverBody', deliverBody) - - - def _deliverBody_INITIAL(self, protocol): - """ - Deliver any buffered data to C{protocol} and prepare to deliver any - future data to it. Move to the C{'CONNECTED'} state. - """ - # Now that there's a protocol to consume the body, resume the - # transport. It was previously paused by HTTPClientParser to avoid - # reading too much data before it could be handled. - self._transport.resumeProducing() - - protocol.makeConnection(self._transport) - self._bodyProtocol = protocol - for data in self._bodyBuffer: - self._bodyProtocol.dataReceived(data) - self._bodyBuffer = None - self._state = 'CONNECTED' - - - def _deliverBody_CONNECTED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol when it is - already being delivered to another protocol. - """ - raise RuntimeError( - "Response already has protocol %r, cannot deliverBody " - "again" % (self._bodyProtocol,)) - - - def _deliverBody_DEFERRED_CLOSE(self, protocol): - """ - Deliver any buffered data to C{protocol} and then disconnect the - protocol. Move to the C{'FINISHED'} state. - """ - # Unlike _deliverBody_INITIAL, there is no need to resume the - # transport here because all of the response data has been received - # already. Some higher level code may want to resume the transport if - # that code expects further data to be received over it. - - protocol.makeConnection(self._transport) - - for data in self._bodyBuffer: - protocol.dataReceived(data) - self._bodyBuffer = None - protocol.connectionLost(self._reason) - self._state = 'FINISHED' - - - def _deliverBody_FINISHED(self, protocol): - """ - It is invalid to attempt to deliver data to a protocol after the - response body has been delivered to another protocol. - """ - raise RuntimeError( - "Response already finished, cannot deliverBody now.") - - - def _bodyDataReceived(self, data): - """ - Called by HTTPClientParser with chunks of data from the response body. - They will be buffered or delivered to the protocol passed to - deliverBody. - """ - _bodyDataReceived = makeStatefulDispatcher('bodyDataReceived', - _bodyDataReceived) - - - def _bodyDataReceived_INITIAL(self, data): - """ - Buffer any data received for later delivery to a protocol passed to - C{deliverBody}. - - Little or no data should be buffered by this method, since the - transport has been paused and will not be resumed until a protocol - is supplied. - """ - self._bodyBuffer.append(data) - - - def _bodyDataReceived_CONNECTED(self, data): - """ - Deliver any data received to the protocol to which this L{Response} - is connected. - """ - self._bodyProtocol.dataReceived(data) - - - def _bodyDataReceived_DEFERRED_CLOSE(self, data): - """ - It is invalid for data to be delivered after it has been indicated - that the response body has been completely delivered. - """ - raise RuntimeError("Cannot receive body data after _bodyDataFinished") - - - def _bodyDataReceived_FINISHED(self, data): - """ - It is invalid for data to be delivered after the response body has - been delivered to a protocol. - """ - raise RuntimeError("Cannot receive body data after protocol disconnected") - - - def _bodyDataFinished(self, reason=None): - """ - Called by HTTPClientParser when no more body data is available. If the - optional reason is supplied, this indicates a problem or potential - problem receiving all of the response body. - """ - _bodyDataFinished = makeStatefulDispatcher('bodyDataFinished', - _bodyDataFinished) - - - def _bodyDataFinished_INITIAL(self, reason=None): - """ - Move to the C{'DEFERRED_CLOSE'} state to wait for a protocol to - which to deliver the response body. - """ - self._state = 'DEFERRED_CLOSE' - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._reason = reason - - - def _bodyDataFinished_CONNECTED(self, reason=None): - """ - Disconnect the protocol and move to the C{'FINISHED'} state. - """ - if reason is None: - reason = Failure(ResponseDone("Response body fully received")) - self._bodyProtocol.connectionLost(reason) - self._bodyProtocol = None - self._state = 'FINISHED' - - - def _bodyDataFinished_DEFERRED_CLOSE(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data more than once") - - - def _bodyDataFinished_FINISHED(self): - """ - It is invalid to attempt to notify the L{Response} of the end of the - response body data more than once. - """ - raise RuntimeError("Cannot finish body data after protocol disconnected") - - - -class ChunkedEncoder: - """ - Helper object which exposes L{IConsumer} on top of L{HTTP11ClientProtocol} - for streaming request bodies to the server. - """ - implements(IConsumer) - - def __init__(self, transport): - self.transport = transport - - - def _allowNoMoreWrites(self): - """ - Indicate that no additional writes are allowed. Attempts to write - after calling this method will be met with an exception. - """ - self.transport = None - - - def registerProducer(self, producer, streaming): - """ - Register the given producer with C{self.transport}. - """ - self.transport.registerProducer(producer, streaming) - - - def write(self, data): - """ - Write the given request body bytes to the transport using chunked - encoding. - - @type data: C{str} - """ - if self.transport is None: - raise ExcessWrite() - self.transport.writeSequence(("%x\r\n" % len(data), data, "\r\n")) - - - def unregisterProducer(self): - """ - Indicate that the request body is complete and finish the request. - """ - self.write('') - self.transport.unregisterProducer() - self._allowNoMoreWrites() - - - -class TransportProxyProducer: - """ - An L{IPushProducer} implementation which wraps another such thing and - proxies calls to it until it is told to stop. - - @ivar _producer: The wrapped L{IPushProducer} provider or C{None} after - this proxy has been stopped. - """ - implements(IPushProducer) - - # LineReceiver uses this undocumented attribute of transports to decide - # when to stop calling lineReceived or rawDataReceived (if it finds it to - # be true, it doesn't bother to deliver any more data). Set disconnecting - # to False here and never change it to true so that all data is always - # delivered to us and so that LineReceiver doesn't fail with an - # AttributeError. - disconnecting = False - - def __init__(self, producer): - self._producer = producer - - - def _stopProxying(self): - """ - Stop forwarding calls of L{IPushProducer} methods to the underlying - L{IPushProvider} provider. - """ - self._producer = None - - - def stopProducing(self): - """ - Proxy the stoppage to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.stopProducing() - - - def resumeProducing(self): - """ - Proxy the resumption to the underlying producer, unless this proxy has - been stopped. - """ - if self._producer is not None: - self._producer.resumeProducing() - - - def pauseProducing(self): - """ - Proxy the pause to the underlying producer, unless this proxy has been - stopped. - """ - if self._producer is not None: - self._producer.pauseProducing() - - - -class HTTP11ClientProtocol(Protocol): - """ - L{HTTP11ClientProtocol} is an implementation of the HTTP 1.1 client - protocol. It supports as few features as possible. - - @ivar _parser: After a request is issued, the L{HTTPClientParser} to - which received data making up the response to that request is - delivered. - - @ivar _finishedRequest: After a request is issued, the L{Deferred} which - will fire when a L{Response} object corresponding to that request is - available. This allows L{HTTP11ClientProtocol} to fail the request - if there is a connection or parsing problem. - - @ivar _currentRequest: After a request is issued, the L{Request} - instance used to make that request. This allows - L{HTTP11ClientProtocol} to stop request generation if necessary (for - example, if the connection is lost). - - @ivar _transportProxy: After a request is issued, the - L{TransportProxyProducer} to which C{_parser} is connected. This - allows C{_parser} to pause and resume the transport in a way which - L{HTTP11ClientProtocol} can exert some control over. - - @ivar _responseDeferred: After a request is issued, the L{Deferred} from - C{_parser} which will fire with a L{Response} when one has been - received. This is eventually chained with C{_finishedRequest}, but - only in certain cases to avoid double firing that Deferred. - - @ivar _state: Indicates what state this L{HTTP11ClientProtocol} instance - is in with respect to transmission of a request and reception of a - response. This may be one of the following strings: - - - QUIESCENT: This is the state L{HTTP11ClientProtocol} instances - start in. Nothing is happening: no request is being sent and no - response is being received or expected. - - - TRANSMITTING: When a request is made (via L{request}), the - instance moves to this state. L{Request.writeTo} has been used - to start to send a request but it has not yet finished. - - - TRANSMITTING_AFTER_RECEIVING_RESPONSE: The server has returned a - complete response but the request has not yet been fully sent - yet. The instance will remain in this state until the request - is fully sent. - - - GENERATION_FAILED: There was an error while the request. The - request was not fully sent to the network. - - - WAITING: The request was fully sent to the network. The - instance is now waiting for the response to be fully received. - - - ABORTING: Application code has requested that the HTTP connection - be aborted. - - - CONNECTION_LOST: The connection has been lost. - - @ivar _abortDeferreds: A list of C{Deferred} instances that will fire when - the connection is lost. - """ - _state = 'QUIESCENT' - _parser = None - _finishedRequest = None - _currentRequest = None - _transportProxy = None - _responseDeferred = None - - - def __init__(self, quiescentCallback=lambda c: None): - self._quiescentCallback = quiescentCallback - self._abortDeferreds = [] - - - @property - def state(self): - return self._state - - - def request(self, request): - """ - Issue C{request} over C{self.transport} and return a L{Deferred} which - will fire with a L{Response} instance or an error. - - @param request: The object defining the parameters of the request to - issue. - @type request: L{Request} - - @rtype: L{Deferred} - @return: The deferred may errback with L{RequestGenerationFailed} if - the request was not fully written to the transport due to a local - error. It may errback with L{RequestTransmissionFailed} if it was - not fully written to the transport due to a network error. It may - errback with L{ResponseFailed} if the request was sent (not - necessarily received) but some or all of the response was lost. It - may errback with L{RequestNotSent} if it is not possible to send - any more requests using this L{HTTP11ClientProtocol}. - """ - if self._state != 'QUIESCENT': - return fail(RequestNotSent()) - - self._state = 'TRANSMITTING' - _requestDeferred = maybeDeferred(request.writeTo, self.transport) - - def cancelRequest(ign): - # Explicitly cancel the request's deferred if it's still trying to - # write when this request is cancelled. - if self._state in ( - 'TRANSMITTING', 'TRANSMITTING_AFTER_RECEIVING_RESPONSE'): - _requestDeferred.cancel() - else: - self.transport.abortConnection() - self._disconnectParser(Failure(CancelledError())) - self._finishedRequest = Deferred(cancelRequest) - - # Keep track of the Request object in case we need to call stopWriting - # on it. - self._currentRequest = request - - self._transportProxy = TransportProxyProducer(self.transport) - self._parser = HTTPClientParser(request, self._finishResponse) - self._parser.makeConnection(self._transportProxy) - self._responseDeferred = self._parser._responseDeferred - - def cbRequestWrotten(ignored): - if self._state == 'TRANSMITTING': - self._state = 'WAITING' - self._responseDeferred.chainDeferred(self._finishedRequest) - - def ebRequestWriting(err): - if self._state == 'TRANSMITTING': - self._state = 'GENERATION_FAILED' - self.transport.abortConnection() - self._finishedRequest.errback( - Failure(RequestGenerationFailed([err]))) - else: - log.err(err, 'Error writing request, but not in valid state ' - 'to finalize request: %s' % self._state) - - _requestDeferred.addCallbacks(cbRequestWrotten, ebRequestWriting) - - return self._finishedRequest - - - def _finishResponse(self, rest): - """ - Called by an L{HTTPClientParser} to indicate that it has parsed a - complete response. - - @param rest: A C{str} giving any trailing bytes which were given to - the L{HTTPClientParser} which were not part of the response it - was parsing. - """ - _finishResponse = makeStatefulDispatcher('finishResponse', _finishResponse) - - - def _finishResponse_WAITING(self, rest): - # Currently the rest parameter is ignored. Don't forget to use it if - # we ever add support for pipelining. And maybe check what trailers - # mean. - if self._state == 'WAITING': - self._state = 'QUIESCENT' - else: - # The server sent the entire response before we could send the - # whole request. That sucks. Oh well. Fire the request() - # Deferred with the response. But first, make sure that if the - # request does ever finish being written that it won't try to fire - # that Deferred. - self._state = 'TRANSMITTING_AFTER_RECEIVING_RESPONSE' - self._responseDeferred.chainDeferred(self._finishedRequest) - - # This will happen if we're being called due to connection being lost; - # if so, no need to disconnect parser again, or to call - # _quiescentCallback. - if self._parser is None: - return - - reason = ConnectionDone("synthetic!") - connHeaders = self._parser.connHeaders.getRawHeaders('connection', ()) - if (('close' in connHeaders) or self._state != "QUIESCENT" or - not self._currentRequest.persistent): - self._giveUp(Failure(reason)) - else: - # We call the quiescent callback first, to ensure connection gets - # added back to connection pool before we finish the request. - try: - self._quiescentCallback(self) - except: - # If callback throws exception, just log it and disconnect; - # keeping persistent connections around is an optimisation: - log.err() - self.transport.loseConnection() - self._disconnectParser(reason) - - - _finishResponse_TRANSMITTING = _finishResponse_WAITING - - - def _disconnectParser(self, reason): - """ - If there is still a parser, call its C{connectionLost} method with the - given reason. If there is not, do nothing. - - @type reason: L{Failure} - """ - if self._parser is not None: - parser = self._parser - self._parser = None - self._currentRequest = None - self._finishedRequest = None - self._responseDeferred = None - - # The parser is no longer allowed to do anything to the real - # transport. Stop proxying from the parser's transport to the real - # transport before telling the parser it's done so that it can't do - # anything. - self._transportProxy._stopProxying() - self._transportProxy = None - parser.connectionLost(reason) - - - def _giveUp(self, reason): - """ - Lose the underlying connection and disconnect the parser with the given - L{Failure}. - - Use this method instead of calling the transport's loseConnection - method directly otherwise random things will break. - """ - self.transport.loseConnection() - self._disconnectParser(reason) - - - def dataReceived(self, bytes): - """ - Handle some stuff from some place. - """ - try: - self._parser.dataReceived(bytes) - except: - self._giveUp(Failure()) - - - def connectionLost(self, reason): - """ - The underlying transport went away. If appropriate, notify the parser - object. - """ - connectionLost = makeStatefulDispatcher('connectionLost', connectionLost) - - - def _connectionLost_QUIESCENT(self, reason): - """ - Nothing is currently happening. Move to the C{'CONNECTION_LOST'} - state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_GENERATION_FAILED(self, reason): - """ - The connection was in an inconsistent state. Move to the - C{'CONNECTION_LOST'} state but otherwise do nothing. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_TRANSMITTING(self, reason): - """ - Fail the L{Deferred} for the current request, notify the request - object that it does not need to continue transmitting itself, and - move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - self._finishedRequest.errback( - Failure(RequestTransmissionFailed([reason]))) - del self._finishedRequest - - # Tell the request that it should stop bothering now. - self._currentRequest.stopWriting() - - - def _connectionLost_TRANSMITTING_AFTER_RECEIVING_RESPONSE(self, reason): - """ - Move to the C{'CONNECTION_LOST'} state. - """ - self._state = 'CONNECTION_LOST' - - - def _connectionLost_WAITING(self, reason): - """ - Disconnect the response parser so that it can propagate the event as - necessary (for example, to call an application protocol's - C{connectionLost} method, or to fail a request L{Deferred}) and move - to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(reason) - self._state = 'CONNECTION_LOST' - - - def _connectionLost_ABORTING(self, reason): - """ - Disconnect the response parser with a L{ConnectionAborted} failure, and - move to the C{'CONNECTION_LOST'} state. - """ - self._disconnectParser(Failure(ConnectionAborted())) - self._state = 'CONNECTION_LOST' - for d in self._abortDeferreds: - d.callback(None) - self._abortDeferreds = [] - - - def abort(self): - """ - Close the connection and cause all outstanding L{request} L{Deferred}s - to fire with an error. - """ - if self._state == "CONNECTION_LOST": - return succeed(None) - self.transport.loseConnection() - self._state = 'ABORTING' - d = Deferred() - self._abortDeferreds.append(d) - return d diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py deleted file mode 100644 index c3830dc47..000000000 --- a/scrapy/xlib/tx/client.py +++ /dev/null @@ -1,1168 +0,0 @@ -# -*- test-case-name: twisted.web.test.test_webclient,twisted.web.test.test_agent -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -HTTP client. -""" - -from __future__ import division, absolute_import - -import os - -try: - from urlparse import urlunparse - from urllib import splithost, splittype -except ImportError: - from urllib.parse import splithost, splittype - from urllib.parse import urlunparse as _urlunparse - - def urlunparse(parts): - result = _urlunparse(tuple([p.decode("charmap") for p in parts])) - return result.encode("charmap") -import zlib - -from zope.interface import implementer - -from twisted.python import log -from twisted.python.failure import Failure -from twisted.web import http -from twisted.internet import defer, protocol, task, reactor -from twisted.internet.interfaces import IProtocol -from twisted.python import failure -from twisted.python.components import proxyForInterface -from twisted.web import error -from twisted.web.http_headers import Headers - -from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer - - -class PartialDownloadError(error.Error): - """ - Page was only partially downloaded, we got disconnected in middle. - - @ivar response: All of the response body which was downloaded. - """ - - -class _URL(tuple): - """ - A parsed URL. - - At some point this should be replaced with a better URL implementation. - """ - def __new__(self, scheme, host, port, path): - return tuple.__new__(_URL, (scheme, host, port, path)) - - - def __init__(self, scheme, host, port, path): - self.scheme = scheme - self.host = host - self.port = port - self.path = path - - -def _parse(url, defaultPort=None): - """ - Split the given URL into the scheme, host, port, and path. - - @type url: C{bytes} - @param url: An URL to parse. - - @type defaultPort: C{int} or C{None} - @param defaultPort: An alternate value to use as the port if the URL does - not include one. - - @return: A four-tuple of the scheme, host, port, and path of the URL. All - of these are C{bytes} instances except for port, which is an C{int}. - """ - url = url.strip() - parsed = http.urlparse(url) - scheme = parsed[0] - path = urlunparse((b'', b'') + parsed[2:]) - - if defaultPort is None: - if scheme == b'https': - defaultPort = 443 - else: - defaultPort = 80 - - host, port = parsed[1], defaultPort - if b':' in host: - host, port = host.split(b':') - try: - port = int(port) - except ValueError: - port = defaultPort - - if path == b'': - path = b'/' - - return _URL(scheme, host, port, path) - - -def _makeGetterFactory(url, factoryFactory, contextFactory=None, - *args, **kwargs): - """ - Create and connect an HTTP page getting factory. - - Any additional positional or keyword arguments are used when calling - C{factoryFactory}. - - @param factoryFactory: Factory factory that is called with C{url}, C{args} - and C{kwargs} to produce the getter - - @param contextFactory: Context factory to use when creating a secure - connection, defaulting to C{None} - - @return: The factory created by C{factoryFactory} - """ - scheme, host, port, path = _parse(url) - factory = factoryFactory(url, *args, **kwargs) - if scheme == b'https': - from twisted.internet import ssl - if contextFactory is None: - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(host, port, factory, contextFactory) - else: - reactor.connectTCP(host, port, factory) - return factory - - -# The code which follows is based on the new HTTP client implementation. It -# should be significantly better than anything above, though it is not yet -# feature equivalent. - -from twisted.web.error import SchemeNotSupported -from ._newclient import Request, Response, HTTP11ClientProtocol -from ._newclient import ResponseDone, ResponseFailed -from ._newclient import RequestNotSent, RequestTransmissionFailed -from ._newclient import ( - ResponseNeverReceived, PotentialDataLoss, _WrapperException) - -try: - from twisted.internet.ssl import ClientContextFactory -except ImportError: - class WebClientContextFactory(object): - """ - A web context factory which doesn't work because the necessary SSL - support is missing. - """ - def getContext(self, hostname, port): - raise NotImplementedError("SSL support unavailable") -else: - class WebClientContextFactory(ClientContextFactory): - """ - A web context factory which ignores the hostname and port and does no - certificate verification. - """ - def getContext(self, hostname, port): - return ClientContextFactory.getContext(self) - - - -class _WebToNormalContextFactory(object): - """ - Adapt a web context factory to a normal context factory. - - @ivar _webContext: A web context factory which accepts a hostname and port - number to its C{getContext} method. - - @ivar _hostname: The hostname which will be passed to - C{_webContext.getContext}. - - @ivar _port: The port number which will be passed to - C{_webContext.getContext}. - """ - def __init__(self, webContext, hostname, port): - self._webContext = webContext - self._hostname = hostname - self._port = port - - - def getContext(self): - """ - Called the wrapped web context factory's C{getContext} method with a - hostname and port number and return the resulting context object. - """ - return self._webContext.getContext(self._hostname, self._port) - - - -@implementer(IBodyProducer) -class FileBodyProducer(object): - """ - L{FileBodyProducer} produces bytes from an input file object incrementally - and writes them to a consumer. - - Since file-like objects cannot be read from in an event-driven manner, - L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from - the file. This process is also paused and resumed based on notifications - from the L{IConsumer} provider being written to. - - The file is closed after it has been read, or if the producer is stopped - early. - - @ivar _inputFile: Any file-like object, bytes read from which will be - written to a consumer. - - @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to - schedule all reads. - - @ivar _readSize: The number of bytes to read from C{_inputFile} at a time. - """ - - # Python 2.4 doesn't have these symbolic constants - _SEEK_SET = getattr(os, 'SEEK_SET', 0) - _SEEK_END = getattr(os, 'SEEK_END', 2) - - def __init__(self, inputFile, cooperator=task, readSize=2 ** 16): - self._inputFile = inputFile - self._cooperate = cooperator.cooperate - self._readSize = readSize - self.length = self._determineLength(inputFile) - - - def _determineLength(self, fObj): - """ - Determine how many bytes can be read out of C{fObj} (assuming it is not - modified from this point on). If the determination cannot be made, - return C{UNKNOWN_LENGTH}. - """ - try: - seek = fObj.seek - tell = fObj.tell - except AttributeError: - return UNKNOWN_LENGTH - originalPosition = tell() - seek(0, self._SEEK_END) - end = tell() - seek(originalPosition, self._SEEK_SET) - return end - originalPosition - - - def stopProducing(self): - """ - Permanently stop writing bytes from the file to the consumer by - stopping the underlying L{CooperativeTask}. - """ - self._inputFile.close() - self._task.stop() - - - def startProducing(self, consumer): - """ - Start a cooperative task which will read bytes from the input file and - write them to C{consumer}. Return a L{Deferred} which fires after all - bytes have been written. - - @param consumer: Any L{IConsumer} provider - """ - self._task = self._cooperate(self._writeloop(consumer)) - d = self._task.whenDone() - def maybeStopped(reason): - # IBodyProducer.startProducing's Deferred isn't support to fire if - # stopProducing is called. - reason.trap(task.TaskStopped) - return defer.Deferred() - d.addCallbacks(lambda ignored: None, maybeStopped) - return d - - - def _writeloop(self, consumer): - """ - Return an iterator which reads one chunk of bytes from the input file - and writes them to the consumer for each time it is iterated. - """ - while True: - bytes = self._inputFile.read(self._readSize) - if not bytes: - self._inputFile.close() - break - consumer.write(bytes) - yield None - - - def pauseProducing(self): - """ - Temporarily suspend copying bytes from the input file to the consumer - by pausing the L{CooperativeTask} which drives that activity. - """ - self._task.pause() - - - def resumeProducing(self): - """ - Undo the effects of a previous C{pauseProducing} and resume copying - bytes to the consumer by resuming the L{CooperativeTask} which drives - the write activity. - """ - self._task.resume() - - - -class _HTTP11ClientFactory(protocol.Factory): - """ - A factory for L{HTTP11ClientProtocol}, used by L{HTTPConnectionPool}. - - @ivar _quiescentCallback: The quiescent callback to be passed to protocol - instances, used to return them to the connection pool. - - @since: 11.1 - """ - def __init__(self, quiescentCallback): - self._quiescentCallback = quiescentCallback - - - def buildProtocol(self, addr): - return HTTP11ClientProtocol(self._quiescentCallback) - - - -class _RetryingHTTP11ClientProtocol(object): - """ - A wrapper for L{HTTP11ClientProtocol} that automatically retries requests. - - @ivar _clientProtocol: The underlying L{HTTP11ClientProtocol}. - - @ivar _newConnection: A callable that creates a new connection for a - retry. - """ - - def __init__(self, clientProtocol, newConnection): - self._clientProtocol = clientProtocol - self._newConnection = newConnection - - - def _shouldRetry(self, method, exception, bodyProducer): - """ - Indicate whether request should be retried. - - Only returns C{True} if method is idempotent, no response was - received, the reason for the failed request was not due to - user-requested cancellation, and no body was sent. The latter - requirement may be relaxed in the future, and PUT added to approved - method list. - """ - if method not in ("GET", "HEAD", "OPTIONS", "DELETE", "TRACE"): - return False - if not isinstance(exception, (RequestNotSent, RequestTransmissionFailed, - ResponseNeverReceived)): - return False - if isinstance(exception, _WrapperException): - for failure in exception.reasons: - if failure.check(defer.CancelledError): - return False - if bodyProducer is not None: - return False - return True - - - def request(self, request): - """ - Do a request, and retry once (with a new connection) it it fails in - a retryable manner. - - @param request: A L{Request} instance that will be requested using the - wrapped protocol. - """ - d = self._clientProtocol.request(request) - - def failed(reason): - if self._shouldRetry(request.method, reason.value, - request.bodyProducer): - return self._newConnection().addCallback( - lambda connection: connection.request(request)) - else: - return reason - d.addErrback(failed) - return d - - - -class HTTPConnectionPool(object): - """ - A pool of persistent HTTP connections. - - Features: - - Cached connections will eventually time out. - - Limits on maximum number of persistent connections. - - Connections are stored using keys, which should be chosen such that any - connections stored under a given key can be used interchangeably. - - Failed requests done using previously cached connections will be retried - once if they use an idempotent method (e.g. GET), in case the HTTP server - timed them out. - - @ivar persistent: Boolean indicating whether connections should be - persistent. Connections are persistent by default. - - @ivar maxPersistentPerHost: The maximum number of cached persistent - connections for a C{host:port} destination. - @type maxPersistentPerHost: C{int} - - @ivar cachedConnectionTimeout: Number of seconds a cached persistent - connection will stay open before disconnecting. - - @ivar retryAutomatically: C{boolean} indicating whether idempotent - requests should be retried once if no response was received. - - @ivar _factory: The factory used to connect to the proxy. - - @ivar _connections: Map (scheme, host, port) to lists of - L{HTTP11ClientProtocol} instances. - - @ivar _timeouts: Map L{HTTP11ClientProtocol} instances to a - C{IDelayedCall} instance of their timeout. - - @since: 12.1 - """ - - _factory = _HTTP11ClientFactory - maxPersistentPerHost = 2 - cachedConnectionTimeout = 240 - retryAutomatically = True - - def __init__(self, reactor, persistent=True): - self._reactor = reactor - self.persistent = persistent - self._connections = {} - self._timeouts = {} - - - def getConnection(self, key, endpoint): - """ - Supply a connection, newly created or retrieved from the pool, to be - used for one HTTP request. - - The connection will remain out of the pool (not available to be - returned from future calls to this method) until one HTTP request has - been completed over it. - - Afterwards, if the connection is still open, it will automatically be - added to the pool. - - @param key: A unique key identifying connections that can be used - interchangeably. - - @param endpoint: An endpoint that can be used to open a new connection - if no cached connection is available. - - @return: A C{Deferred} that will fire with a L{HTTP11ClientProtocol} - (or a wrapper) that can be used to send a single HTTP request. - """ - # Try to get cached version: - connections = self._connections.get(key) - while connections: - connection = connections.pop(0) - # Cancel timeout: - self._timeouts[connection].cancel() - del self._timeouts[connection] - if connection.state == "QUIESCENT": - if self.retryAutomatically: - newConnection = lambda: self._newConnection(key, endpoint) - connection = _RetryingHTTP11ClientProtocol( - connection, newConnection) - return defer.succeed(connection) - - return self._newConnection(key, endpoint) - - - def _newConnection(self, key, endpoint): - """ - Create a new connection. - - This implements the new connection code path for L{getConnection}. - """ - def quiescentCallback(protocol): - self._putConnection(key, protocol) - factory = self._factory(quiescentCallback) - return endpoint.connect(factory) - - - def _removeConnection(self, key, connection): - """ - Remove a connection from the cache and disconnect it. - """ - connection.transport.loseConnection() - self._connections[key].remove(connection) - del self._timeouts[connection] - - - def _putConnection(self, key, connection): - """ - Return a persistent connection to the pool. This will be called by - L{HTTP11ClientProtocol} when the connection becomes quiescent. - """ - if connection.state != "QUIESCENT": - # Log with traceback for debugging purposes: - try: - raise RuntimeError( - "BUG: Non-quiescent protocol added to connection pool.") - except: - log.err() - return - connections = self._connections.setdefault(key, []) - if len(connections) == self.maxPersistentPerHost: - dropped = connections.pop(0) - dropped.transport.loseConnection() - self._timeouts[dropped].cancel() - del self._timeouts[dropped] - connections.append(connection) - cid = self._reactor.callLater(self.cachedConnectionTimeout, - self._removeConnection, - key, connection) - self._timeouts[connection] = cid - - - def closeCachedConnections(self): - """ - Close all persistent connections and remove them from the pool. - - @return: L{defer.Deferred} that fires when all connections have been - closed. - """ - results = [] - for protocols in self._connections.itervalues(): - for p in protocols: - results.append(p.abort()) - self._connections = {} - for dc in self._timeouts.values(): - dc.cancel() - self._timeouts = {} - return defer.gatherResults(results).addCallback(lambda ign: None) - - - -class _AgentBase(object): - """ - Base class offering common facilities for L{Agent}-type classes. - - @ivar _reactor: The C{IReactorTime} implementation which will be used by - the pool, and perhaps by subclasses as well. - - @ivar _pool: The L{HTTPConnectionPool} used to manage HTTP connections. - """ - - def __init__(self, reactor, pool): - if pool is None: - pool = HTTPConnectionPool(reactor, False) - self._reactor = reactor - self._pool = pool - - - def _computeHostValue(self, scheme, host, port): - """ - Compute the string to use for the value of the I{Host} header, based on - the given scheme, host name, and port number. - """ - if (scheme, port) in (('http', 80), ('https', 443)): - return host - return '%s:%d' % (host, port) - - - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath): - """ - Issue a new request, given the endpoint and the path sent as part of - the request. - """ - # Create minimal headers, if necessary: - if headers is None: - headers = Headers() - if not headers.hasHeader('host'): - #headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us - headers.addRawHeader( - 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, - parsedURI.port)) - - d = self._pool.getConnection(key, endpoint) - def cbConnected(proto): - return proto.request( - Request(method, requestPath, headers, bodyProducer, - persistent=self._pool.persistent)) - d.addCallback(cbConnected) - return d - - - -class Agent(_AgentBase): - """ - L{Agent} is a very basic HTTP client. It supports I{HTTP} and I{HTTPS} - scheme URIs (but performs no certificate checking by default). - - @param pool: A L{HTTPConnectionPool} instance, or C{None}, in which case a - non-persistent L{HTTPConnectionPool} instance will be created. - - @ivar _contextFactory: A web context factory which will be used to create - SSL context objects for any SSL connections the agent needs to make. - - @ivar _connectTimeout: If not C{None}, the timeout passed to C{connectTCP} - or C{connectSSL} for specifying the connection timeout. - - @ivar _bindAddress: If not C{None}, the address passed to C{connectTCP} or - C{connectSSL} for specifying the local address to bind to. - - @since: 9.0 - """ - - def __init__(self, reactor, contextFactory=WebClientContextFactory(), - connectTimeout=None, bindAddress=None, - pool=None): - _AgentBase.__init__(self, reactor, pool) - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - - - def _wrapContextFactory(self, host, port): - """ - Create and return a normal context factory wrapped around - C{self._contextFactory} in such a way that C{self._contextFactory} will - have the host and port information passed to it. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: A context factory suitable to be passed to - C{reactor.connectSSL}. - """ - return _WebToNormalContextFactory(self._contextFactory, host, port) - - - def _getEndpoint(self, scheme, host, port): - """ - Get an endpoint for the given host and port, using a transport - selected based on scheme. - - @param scheme: A string like C{'http'} or C{'https'} (the only two - supported values) to use to determine how to establish the - connection. - - @param host: A C{str} giving the hostname which will be connected to in - order to issue a request. - - @param port: An C{int} giving the port number the connection will be - on. - - @return: An endpoint which can be used to connect to given address. - """ - kwargs = {} - if self._connectTimeout is not None: - kwargs['timeout'] = self._connectTimeout - kwargs['bindAddress'] = self._bindAddress - if scheme == 'http': - return TCP4ClientEndpoint(self._reactor, host, port, **kwargs) - elif scheme == 'https': - return SSL4ClientEndpoint(self._reactor, host, port, - self._wrapContextFactory(host, port), - **kwargs) - else: - raise SchemeNotSupported("Unsupported scheme: %r" % (scheme,)) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request. - - @param method: The request method to send. - @type method: C{str} - - @param uri: The request URI send. - @type uri: C{str} - - @param headers: The request headers to send. If no I{Host} header is - included, one will be added based on the request URI. - @type headers: L{Headers} - - @param bodyProducer: An object which will produce the request body or, - if the request body is to be empty, L{None}. - @type bodyProducer: L{IBodyProducer} provider - - @return: A L{Deferred} which fires with the result of the request (a - L{twisted.web.iweb.IResponse} provider), or fails if there is a - problem setting up a connection over which to issue the request. - It may also fail with L{SchemeNotSupported} if the scheme of the - given URI is not supported. - @rtype: L{Deferred} - """ - parsedURI = _parse(uri) - try: - endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host, - parsedURI.port) - except SchemeNotSupported: - return defer.fail(Failure()) - key = (parsedURI.scheme, parsedURI.host, parsedURI.port) - return self._requestWithEndpoint(key, endpoint, method, parsedURI, - headers, bodyProducer, parsedURI.path) - - - -class ProxyAgent(_AgentBase): - """ - An HTTP agent able to cross HTTP proxies. - - @ivar _proxyEndpoint: The endpoint used to connect to the proxy. - - @since: 11.1 - """ - - def __init__(self, endpoint, reactor=None, pool=None): - if reactor is None: - from twisted.internet import reactor - _AgentBase.__init__(self, reactor, pool) - self._proxyEndpoint = endpoint - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request via the configured proxy. - """ - # Cache *all* connections under the same key, since we are only - # connecting to a single destination, the proxy: - key = ("http-proxy", self._proxyEndpoint) - - # To support proxying HTTPS via CONNECT, we will use key - # ("http-proxy-CONNECT", scheme, host, port), and an endpoint that - # wraps _proxyEndpoint with an additional callback to do the CONNECT. - return self._requestWithEndpoint(key, self._proxyEndpoint, method, - _parse(uri), headers, bodyProducer, - uri) - - - -class _FakeUrllib2Request(object): - """ - A fake C{urllib2.Request} object for C{cookielib} to work with. - - @see: U{http://docs.python.org/library/urllib2.html#request-objects} - - @type uri: C{str} - @ivar uri: Request URI. - - @type headers: L{twisted.web.http_headers.Headers} - @ivar headers: Request headers. - - @type type: C{str} - @ivar type: The scheme of the URI. - - @type host: C{str} - @ivar host: The host[:port] of the URI. - - @since: 11.1 - """ - def __init__(self, uri): - self.uri = uri - self.headers = Headers() - self.type, rest = splittype(self.uri) - self.host, rest = splithost(rest) - - - def has_header(self, header): - return self.headers.hasHeader(header) - - - def add_unredirected_header(self, name, value): - self.headers.addRawHeader(name, value) - - - def get_full_url(self): - return self.uri - - - def get_header(self, name, default=None): - headers = self.headers.getRawHeaders(name, default) - if headers is not None: - return headers[0] - return None - - - def get_host(self): - return self.host - - - def get_type(self): - return self.type - - - def is_unverifiable(self): - # In theory this shouldn't be hardcoded. - return False - - - -class _FakeUrllib2Response(object): - """ - A fake C{urllib2.Response} object for C{cookielib} to work with. - - @type response: C{twisted.web.iweb.IResponse} - @ivar response: Underlying Twisted Web response. - - @since: 11.1 - """ - def __init__(self, response): - self.response = response - - - def info(self): - class _Meta(object): - def getheaders(zelf, name): - return self.response.headers.getRawHeaders(name, []) - return _Meta() - - - -class CookieAgent(object): - """ - L{CookieAgent} extends the basic L{Agent} to add RFC-compliant - handling of HTTP cookies. Cookies are written to and extracted - from a C{cookielib.CookieJar} instance. - - The same cookie jar instance will be used for any requests through this - agent, mutating it whenever a I{Set-Cookie} header appears in a response. - - @type _agent: L{twisted.web.client.Agent} - @ivar _agent: Underlying Twisted Web agent to issue requests through. - - @type cookieJar: C{cookielib.CookieJar} - @ivar cookieJar: Initialized cookie jar to read cookies from and store - cookies to. - - @since: 11.1 - """ - def __init__(self, agent, cookieJar): - self._agent = agent - self.cookieJar = cookieJar - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Issue a new request to the wrapped L{Agent}. - - Send a I{Cookie} header if a cookie for C{uri} is stored in - L{CookieAgent.cookieJar}. Cookies are automatically extracted and - stored from requests. - - If a C{'cookie'} header appears in C{headers} it will override the - automatic cookie header obtained from the cookie jar. - - @see: L{Agent.request} - """ - if headers is None: - headers = Headers() - lastRequest = _FakeUrllib2Request(uri) - # Setting a cookie header explicitly will disable automatic request - # cookies. - if not headers.hasHeader('cookie'): - self.cookieJar.add_cookie_header(lastRequest) - cookieHeader = lastRequest.get_header('Cookie', None) - if cookieHeader is not None: - headers = headers.copy() - headers.addRawHeader('cookie', cookieHeader) - - d = self._agent.request(method, uri, headers, bodyProducer) - d.addCallback(self._extractCookies, lastRequest) - return d - - - def _extractCookies(self, response, request): - """ - Extract response cookies and store them in the cookie jar. - - @type response: L{twisted.web.iweb.IResponse} - @param response: Twisted Web response. - - @param request: A urllib2 compatible request object. - """ - resp = _FakeUrllib2Response(response) - self.cookieJar.extract_cookies(resp, request) - return response - - - -class GzipDecoder(proxyForInterface(IResponse)): - """ - A wrapper for a L{Response} instance which handles gzip'ed body. - - @ivar original: The original L{Response} object. - - @since: 11.1 - """ - - def __init__(self, response): - self.original = response - self.length = UNKNOWN_LENGTH - - - def deliverBody(self, protocol): - """ - Override C{deliverBody} to wrap the given C{protocol} with - L{_GzipProtocol}. - """ - self.original.deliverBody(_GzipProtocol(protocol, self.original)) - - - -class _GzipProtocol(proxyForInterface(IProtocol)): - """ - A L{Protocol} implementation which wraps another one, transparently - decompressing received data. - - @ivar _zlibDecompress: A zlib decompress object used to decompress the data - stream. - - @ivar _response: A reference to the original response, in case of errors. - - @since: 11.1 - """ - - def __init__(self, protocol, response): - self.original = protocol - self._response = response - self._zlibDecompress = zlib.decompressobj(16 + zlib.MAX_WBITS) - - - def dataReceived(self, data): - """ - Decompress C{data} with the zlib decompressor, forwarding the raw data - to the original protocol. - """ - try: - rawData = self._zlibDecompress.decompress(data) - except zlib.error: - raise ResponseFailed([failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - - - def connectionLost(self, reason): - """ - Forward the connection lost event, flushing remaining data from the - decompressor if any. - """ - try: - rawData = self._zlibDecompress.flush() - except zlib.error: - raise ResponseFailed([reason, failure.Failure()], self._response) - if rawData: - self.original.dataReceived(rawData) - self.original.connectionLost(reason) - - - -class ContentDecoderAgent(object): - """ - An L{Agent} wrapper to handle encoded content. - - It takes care of declaring the support for content in the - I{Accept-Encoding} header, and automatically decompresses the received data - if it's effectively using compression. - - @param decoders: A list or tuple of (name, decoder) objects. The name - declares which decoding the decoder supports, and the decoder must - return a response object when called/instantiated. For example, - C{(('gzip', GzipDecoder))}. The order determines how the decoders are - going to be advertized to the server. - - @since: 11.1 - """ - - def __init__(self, agent, decoders): - self._agent = agent - self._decoders = dict(decoders) - self._supported = ','.join([decoder[0] for decoder in decoders]) - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request which declares supporting compressed content. - - @see: L{Agent.request}. - """ - if headers is None: - headers = Headers() - else: - headers = headers.copy() - headers.addRawHeader('accept-encoding', self._supported) - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback(self._handleResponse) - - - def _handleResponse(self, response): - """ - Check if the response is encoded, and wrap it to handle decompression. - """ - contentEncodingHeaders = response.headers.getRawHeaders( - 'content-encoding', []) - contentEncodingHeaders = ','.join(contentEncodingHeaders).split(',') - while contentEncodingHeaders: - name = contentEncodingHeaders.pop().strip() - decoder = self._decoders.get(name) - if decoder is not None: - response = decoder(response) - else: - # Add it back - contentEncodingHeaders.append(name) - break - if contentEncodingHeaders: - response.headers.setRawHeaders( - 'content-encoding', [','.join(contentEncodingHeaders)]) - else: - response.headers.removeHeader('content-encoding') - return response - - - -class RedirectAgent(object): - """ - An L{Agent} wrapper which handles HTTP redirects. - - The implementation is rather strict: 301 and 302 behaves like 307, not - redirecting automatically on methods different from C{GET} and C{HEAD}. - - @param redirectLimit: The maximum number of times the agent is allowed to - follow redirects before failing with a L{error.InfiniteRedirection}. - - @since: 11.1 - """ - - def __init__(self, agent, redirectLimit=20): - self._agent = agent - self._redirectLimit = redirectLimit - - - def request(self, method, uri, headers=None, bodyProducer=None): - """ - Send a client request following HTTP redirects. - - @see: L{Agent.request}. - """ - deferred = self._agent.request(method, uri, headers, bodyProducer) - return deferred.addCallback( - self._handleResponse, method, uri, headers, 0) - - - def _handleRedirect(self, response, method, uri, headers, redirectCount): - """ - Handle a redirect response, checking the number of redirects already - followed, and extracting the location header fields. - """ - if redirectCount >= self._redirectLimit: - err = error.InfiniteRedirection( - response.code, - 'Infinite redirection detected', - location=uri) - raise ResponseFailed([failure.Failure(err)], response) - locationHeaders = response.headers.getRawHeaders('location', []) - if not locationHeaders: - err = error.RedirectWithNoLocation( - response.code, 'No location header field', uri) - raise ResponseFailed([failure.Failure(err)], response) - location = locationHeaders[0] - deferred = self._agent.request(method, location, headers) - return deferred.addCallback( - self._handleResponse, method, uri, headers, redirectCount + 1) - - - def _handleResponse(self, response, method, uri, headers, redirectCount): - """ - Handle the response, making another request if it indicates a redirect. - """ - if response.code in (http.MOVED_PERMANENTLY, http.FOUND, - http.TEMPORARY_REDIRECT): - if method not in ('GET', 'HEAD'): - err = error.PageRedirect(response.code, location=uri) - raise ResponseFailed([failure.Failure(err)], response) - return self._handleRedirect(response, method, uri, headers, - redirectCount) - elif response.code == http.SEE_OTHER: - return self._handleRedirect(response, 'GET', uri, headers, - redirectCount) - return response - - - -class _ReadBodyProtocol(protocol.Protocol): - """ - Protocol that collects data sent to it. - - This is a helper for L{IResponse.deliverBody}, which collects the body and - fires a deferred with it. - - @ivar deferred: See L{__init__}. - @ivar status: See L{__init__}. - @ivar message: See L{__init__}. - - @ivar dataBuffer: list of byte-strings received - @type dataBuffer: L{list} of L{bytes} - """ - - def __init__(self, status, message, deferred): - """ - @param status: Status of L{IResponse} - @ivar status: L{int} - - @param message: Message of L{IResponse} - @type message: L{bytes} - - @param deferred: deferred to fire when response is complete - @type deferred: L{Deferred} firing with L{bytes} - """ - self.deferred = deferred - self.status = status - self.message = message - self.dataBuffer = [] - - - def dataReceived(self, data): - """ - Accumulate some more bytes from the response. - """ - self.dataBuffer.append(data) - - - def connectionLost(self, reason): - """ - Deliver the accumulated response bytes to the waiting L{Deferred}, if - the response body has been completely received without error. - """ - if reason.check(ResponseDone): - self.deferred.callback(b''.join(self.dataBuffer)) - elif reason.check(PotentialDataLoss): - self.deferred.errback( - PartialDownloadError(self.status, self.message, - b''.join(self.dataBuffer))) - else: - self.deferred.errback(reason) - - - -def readBody(response): - """ - Get the body of an L{IResponse} and return it as a byte string. - - This is a helper function for clients that don't want to incrementally - receive the body of an HTTP response. - - @param response: The HTTP response for which the body will be read. - @type response: L{IResponse} provider - - @return: A L{Deferred} which will fire with the body of the response. - """ - d = defer.Deferred() - response.deliverBody(_ReadBodyProtocol(response.code, response.phrase, d)) - return d - - - -__all__ = [ - 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', - 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', - 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', - 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', - 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py deleted file mode 100644 index d8a92ccd0..000000000 --- a/scrapy/xlib/tx/endpoints.py +++ /dev/null @@ -1,1269 +0,0 @@ -# -*- test-case-name: twisted.internet.test.test_endpoints -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Implementations of L{IStreamServerEndpoint} and L{IStreamClientEndpoint} that -wrap the L{IReactorTCP}, L{IReactorSSL}, and L{IReactorUNIX} interfaces. - -This also implements an extensible mini-language for describing endpoints, -parsed by the L{clientFromString} and L{serverFromString} functions. - -@since: 10.1 -""" - -from __future__ import division, absolute_import - -import os -import socket - -from zope.interface import implementer, directlyProvides -import warnings - -from twisted.internet import interfaces, defer, error, fdesc, threads -from twisted.internet.protocol import ( - ClientFactory, Protocol, ProcessProtocol, Factory) -from twisted.internet.interfaces import IStreamServerEndpointStringParser -from twisted.internet.interfaces import IStreamClientEndpointStringParser -from twisted.python.filepath import FilePath -from twisted.python.failure import Failure -from twisted.python import log -from twisted.python.components import proxyForInterface - -from twisted.plugin import IPlugin, getPlugins -from twisted.internet import stdio - -from .interfaces import IFileDescriptorReceiver - - -__all__ = ["TCP4ClientEndpoint", "SSL4ServerEndpoint"] - - -class _WrappingProtocol(Protocol): - """ - Wrap another protocol in order to notify my user when a connection has - been made. - """ - - def __init__(self, connectedDeferred, wrappedProtocol): - """ - @param connectedDeferred: The L{Deferred} that will callback - with the C{wrappedProtocol} when it is connected. - - @param wrappedProtocol: An L{IProtocol} provider that will be - connected. - """ - self._connectedDeferred = connectedDeferred - self._wrappedProtocol = wrappedProtocol - - for iface in [interfaces.IHalfCloseableProtocol, - IFileDescriptorReceiver]: - if iface.providedBy(self._wrappedProtocol): - directlyProvides(self, iface) - - - def logPrefix(self): - """ - Transparently pass through the wrapped protocol's log prefix. - """ - if interfaces.ILoggingContext.providedBy(self._wrappedProtocol): - return self._wrappedProtocol.logPrefix() - return self._wrappedProtocol.__class__.__name__ - - - def connectionMade(self): - """ - Connect the C{self._wrappedProtocol} to our C{self.transport} and - callback C{self._connectedDeferred} with the C{self._wrappedProtocol} - """ - self._wrappedProtocol.makeConnection(self.transport) - self._connectedDeferred.callback(self._wrappedProtocol) - - - def dataReceived(self, data): - """ - Proxy C{dataReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.dataReceived(data) - - - def fileDescriptorReceived(self, descriptor): - """ - Proxy C{fileDescriptorReceived} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.fileDescriptorReceived(descriptor) - - - def connectionLost(self, reason): - """ - Proxy C{connectionLost} calls to our C{self._wrappedProtocol} - """ - return self._wrappedProtocol.connectionLost(reason) - - - def readConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.readConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.readConnectionLost() - - - def writeConnectionLost(self): - """ - Proxy L{IHalfCloseableProtocol.writeConnectionLost} to our - C{self._wrappedProtocol} - """ - self._wrappedProtocol.writeConnectionLost() - - - -class _WrappingFactory(ClientFactory): - """ - Wrap a factory in order to wrap the protocols it builds. - - @ivar _wrappedFactory: A provider of I{IProtocolFactory} whose buildProtocol - method will be called and whose resulting protocol will be wrapped. - - @ivar _onConnection: A L{Deferred} that fires when the protocol is - connected - - @ivar _connector: A L{connector } - that is managing the current or previous connection attempt. - """ - protocol = _WrappingProtocol - - def __init__(self, wrappedFactory): - """ - @param wrappedFactory: A provider of I{IProtocolFactory} whose - buildProtocol method will be called and whose resulting protocol - will be wrapped. - """ - self._wrappedFactory = wrappedFactory - self._onConnection = defer.Deferred(canceller=self._canceller) - - - def startedConnecting(self, connector): - """ - A connection attempt was started. Remember the connector which started - said attempt, for use later. - """ - self._connector = connector - - - def _canceller(self, deferred): - """ - The outgoing connection attempt was cancelled. Fail that L{Deferred} - with an L{error.ConnectingCancelledError}. - - @param deferred: The L{Deferred } that was cancelled; - should be the same as C{self._onConnection}. - @type deferred: L{Deferred } - - @note: This relies on startedConnecting having been called, so it may - seem as though there's a race condition where C{_connector} may not - have been set. However, using public APIs, this condition is - impossible to catch, because a connection API - (C{connectTCP}/C{SSL}/C{UNIX}) is always invoked before a - L{_WrappingFactory}'s L{Deferred } is returned to - C{connect()}'s caller. - - @return: C{None} - """ - deferred.errback( - error.ConnectingCancelledError( - self._connector.getDestination())) - self._connector.stopConnecting() - - - def doStart(self): - """ - Start notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStart() - - - def doStop(self): - """ - Stop notifications are passed straight through to the wrapped factory. - """ - self._wrappedFactory.doStop() - - - def buildProtocol(self, addr): - """ - Proxy C{buildProtocol} to our C{self._wrappedFactory} or errback - the C{self._onConnection} L{Deferred}. - - @return: An instance of L{_WrappingProtocol} or C{None} - """ - try: - proto = self._wrappedFactory.buildProtocol(addr) - except: - self._onConnection.errback() - else: - return self.protocol(self._onConnection, proto) - - - def clientConnectionFailed(self, connector, reason): - """ - Errback the C{self._onConnection} L{Deferred} when the - client connection fails. - """ - if not self._onConnection.called: - self._onConnection.errback(reason) - - - - - -@implementer(interfaces.ITransport) -class _ProcessEndpointTransport(proxyForInterface( - interfaces.IProcessTransport, '_process')): - """ - An L{ITransport} provider for the L{IProtocol} instance passed to the - process endpoint. - - @ivar _process: An active process transport which will be used by write - methods on this object to write data to a child process. - @type _process: L{interfaces.IProcessTransport} provider - """ - - def write(self, data): - """ - Write to the child process's standard input. - - @param data: The data to write on stdin. - """ - self._process.writeToChild(0, data) - - - def writeSequence(self, data): - """ - Write a list of strings to child process's stdin. - - @param data: The list of chunks to write on stdin. - """ - for chunk in data: - self._process.writeToChild(0, chunk) - - -@implementer(interfaces.IStreamServerEndpoint) -class _TCPServerEndpoint(object): - """ - A TCP server endpoint interface - """ - - def __init__(self, reactor, port, backlog, interface): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - self._reactor = reactor - self._port = port - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a TCP - socket - """ - return defer.execute(self._reactor.listenTCP, - self._port, - protocolFactory, - backlog=self._backlog, - interface=self._interface) - - - -class TCP4ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv4 configuration - """ - def __init__(self, reactor, port, backlog=50, interface=''): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -class TCP6ServerEndpoint(_TCPServerEndpoint): - """ - Implements TCP server endpoint with an IPv6 configuration - """ - def __init__(self, reactor, port, backlog=50, interface='::'): - """ - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - _TCPServerEndpoint.__init__(self, reactor, port, backlog, interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class TCP4ClientEndpoint(object): - """ - TCP client endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, host, port, timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorTCP} provider - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param timeout: The number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectTCP( - self._host, self._port, wf, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - - -@implementer(interfaces.IStreamServerEndpoint) -class SSL4ServerEndpoint(object): - """ - SSL secured TCP server endpoint with an IPv4 configuration. - """ - - def __init__(self, reactor, port, sslContextFactory, - backlog=50, interface=''): - """ - @param reactor: An L{IReactorSSL} provider. - - @param port: The port number used for listening - @type port: int - - @param sslContextFactory: An instance of - L{twisted.internet.ssl.ContextFactory}. - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to, defaults to '' (all) - @type interface: str - """ - self._reactor = reactor - self._port = port - self._sslContextFactory = sslContextFactory - self._backlog = backlog - self._interface = interface - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen for SSL on a - TCP socket. - """ - return defer.execute(self._reactor.listenSSL, self._port, - protocolFactory, - contextFactory=self._sslContextFactory, - backlog=self._backlog, - interface=self._interface) - - - -@implementer(interfaces.IStreamClientEndpoint) -class SSL4ClientEndpoint(object): - """ - SSL secured TCP client endpoint with an IPv4 configuration - """ - - def __init__(self, reactor, host, port, sslContextFactory, - timeout=30, bindAddress=None): - """ - @param reactor: An L{IReactorSSL} provider. - - @param host: A hostname, used when connecting - @type host: str - - @param port: The port number, used when connecting - @type port: int - - @param sslContextFactory: SSL Configuration information as an instance - of L{twisted.internet.ssl.ContextFactory}. - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param bindAddress: A (host, port) tuple of local address to bind to, - or None. - @type bindAddress: tuple - """ - self._reactor = reactor - self._host = host - self._port = port - self._sslContextFactory = sslContextFactory - self._timeout = timeout - self._bindAddress = bindAddress - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect with SSL over - TCP. - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectSSL( - self._host, self._port, wf, self._sslContextFactory, - timeout=self._timeout, bindAddress=self._bindAddress) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class UNIXServerEndpoint(object): - """ - UnixSocket server endpoint. - """ - def __init__(self, reactor, address, backlog=50, mode=0o666, wantPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - @param address: The path to the Unix socket file, used when listening - @param backlog: number of connections to allow in backlog. - @param mode: mode to set on the unix socket. This parameter is - deprecated. Permissions should be set on the directory which - contains the UNIX socket. - @param wantPID: If True, create a pidfile for the socket. - """ - self._reactor = reactor - self._address = address - self._backlog = backlog - self._mode = mode - self._wantPID = wantPID - - - def listen(self, protocolFactory): - """ - Implement L{IStreamServerEndpoint.listen} to listen on a UNIX socket. - """ - return defer.execute(self._reactor.listenUNIX, self._address, - protocolFactory, - backlog=self._backlog, - mode=self._mode, - wantPID=self._wantPID) - - - -@implementer(interfaces.IStreamClientEndpoint) -class UNIXClientEndpoint(object): - """ - UnixSocket client endpoint. - """ - def __init__(self, reactor, path, timeout=30, checkPID=0): - """ - @param reactor: An L{IReactorUNIX} provider. - - @param path: The path to the Unix socket file, used when connecting - @type path: str - - @param timeout: Number of seconds to wait before assuming the - connection has failed. - @type timeout: int - - @param checkPID: If True, check for a pid file to verify that a server - is listening. - @type checkPID: bool - """ - self._reactor = reactor - self._path = path - self._timeout = timeout - self._checkPID = checkPID - - - def connect(self, protocolFactory): - """ - Implement L{IStreamClientEndpoint.connect} to connect via a - UNIX Socket - """ - try: - wf = _WrappingFactory(protocolFactory) - self._reactor.connectUNIX( - self._path, wf, - timeout=self._timeout, - checkPID=self._checkPID) - return wf._onConnection - except: - return defer.fail() - - - -@implementer(interfaces.IStreamServerEndpoint) -class AdoptedStreamServerEndpoint(object): - """ - An endpoint for listening on a file descriptor initialized outside of - Twisted. - - @ivar _used: A C{bool} indicating whether this endpoint has been used to - listen with a factory yet. C{True} if so. - """ - _close = os.close - _setNonBlocking = staticmethod(fdesc.setNonBlocking) - - def __init__(self, reactor, fileno, addressFamily): - """ - @param reactor: An L{IReactorSocket} provider. - - @param fileno: An integer file descriptor corresponding to a listening - I{SOCK_STREAM} socket. - - @param addressFamily: The address family of the socket given by - C{fileno}. - """ - self.reactor = reactor - self.fileno = fileno - self.addressFamily = addressFamily - self._used = False - - - def listen(self, factory): - """ - Implement L{IStreamServerEndpoint.listen} to start listening on, and - then close, C{self._fileno}. - """ - if self._used: - return defer.fail(error.AlreadyListened()) - self._used = True - - try: - self._setNonBlocking(self.fileno) - port = self.reactor.adoptStreamPort( - self.fileno, self.addressFamily, factory) - self._close(self.fileno) - except: - return defer.fail() - return defer.succeed(port) - - - -def _parseTCP(factory, port, interface="", backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a TCP(IPv4) stream endpoint into the structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenTCP} (or, modulo argument 2, the factory, arguments - to L{TCP4ServerEndpoint}. - """ - return (int(port), factory), {'interface': interface, - 'backlog': int(backlog)} - - - -def _parseUNIX(factory, address, mode='666', backlog=50, lockfile=True): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for a UNIX (AF_UNIX/SOCK_STREAM) stream endpoint into the - structured arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - - @type factory: L{IProtocolFactory} or C{NoneType} - - @param address: the pathname of the unix socket - @type address: C{str} - - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param lockfile: A string '0' or '1', mapping to True and False - respectively. See the C{wantPID} argument to C{listenUNIX} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorTCP.listenUNIX} (or, modulo argument 2, the factory, - arguments to L{UNIXServerEndpoint}. - """ - return ( - (address, factory), - {'mode': int(mode, 8), 'backlog': int(backlog), - 'wantPID': bool(int(lockfile))}) - - - -def _parseSSL(factory, port, privateKey="server.pem", certKey=None, - sslmethod=None, interface='', backlog=50): - """ - Internal parser function for L{_parseServer} to convert the string - arguments for an SSL (over TCP/IPv4) stream endpoint into the structured - arguments. - - @param factory: the protocol factory being parsed, or C{None}. (This was a - leftover argument from when this code was in C{strports}, and is now - mostly None and unused.) - @type factory: L{IProtocolFactory} or C{NoneType} - - @param port: the integer port number to bind - @type port: C{str} - - @param interface: the interface IP to listen on - @param backlog: the length of the listen queue - @type backlog: C{str} - - @param privateKey: The file name of a PEM format private key file. - @type privateKey: C{str} - - @param certKey: The file name of a PEM format certificate file. - @type certKey: C{str} - - @param sslmethod: The string name of an SSL method, based on the name of a - constant in C{OpenSSL.SSL}. Must be one of: "SSLv23_METHOD", - "SSLv2_METHOD", "SSLv3_METHOD", "TLSv1_METHOD". - @type sslmethod: C{str} - - @return: a 2-tuple of (args, kwargs), describing the parameters to - L{IReactorSSL.listenSSL} (or, modulo argument 2, the factory, arguments - to L{SSL4ServerEndpoint}. - """ - from twisted.internet import ssl - if certKey is None: - certKey = privateKey - kw = {} - if sslmethod is not None: - kw['method'] = getattr(ssl.SSL, sslmethod) - else: - kw['method'] = ssl.SSL.SSLv23_METHOD - certPEM = FilePath(certKey).getContent() - keyPEM = FilePath(privateKey).getContent() - privateCertificate = ssl.PrivateCertificate.loadPEM(certPEM + keyPEM) - cf = ssl.CertificateOptions( - privateKey=privateCertificate.privateKey.original, - certificate=privateCertificate.original, - **kw - ) - return ((int(port), factory, cf), - {'interface': interface, 'backlog': int(backlog)}) - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _StandardIOParser(object): - """ - Stream server endpoint string parser for the Standard I/O type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "stdio" - - def _parseServer(self, reactor): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{StandardIOEndpoint} - - @param reactor: Reactor for the endpoint - """ - return StandardIOEndpoint(reactor) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor) - - - - -@implementer(IPlugin, IStreamServerEndpointStringParser) -class _TCP6ServerParser(object): - """ - Stream server endpoint string parser for the TCP6ServerEndpoint type. - - @ivar prefix: See L{IStreamClientEndpointStringParser.prefix}. - """ - prefix = "tcp6" # Used in _parseServer to identify the plugin with the endpoint type - - def _parseServer(self, reactor, port, backlog=50, interface='::'): - """ - Internal parser function for L{_parseServer} to convert the string - arguments into structured arguments for the L{TCP6ServerEndpoint} - - @param reactor: An L{IReactorTCP} provider. - - @param port: The port number used for listening - @type port: int - - @param backlog: Size of the listen queue - @type backlog: int - - @param interface: The hostname to bind to - @type interface: str - """ - port = int(port) - backlog = int(backlog) - return TCP6ServerEndpoint(reactor, port, backlog, interface) - - - def parseStreamServer(self, reactor, *args, **kwargs): - # Redirects to another function (self._parseServer), tricks zope.interface - # into believing the interface is correctly implemented. - return self._parseServer(reactor, *args, **kwargs) - - - -_serverParsers = {"tcp": _parseTCP, - "unix": _parseUNIX, - "ssl": _parseSSL, - } - -_OP, _STRING = range(2) - -def _tokenize(description): - """ - Tokenize a strports string and yield each token. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: an iterable of 2-tuples of (L{_OP} or L{_STRING}, string). Tuples - starting with L{_OP} will contain a second element of either ':' (i.e. - 'next parameter') or '=' (i.e. 'assign parameter value'). For example, - the string 'hello:greet\=ing=world' would result in a generator - yielding these values:: - - _STRING, 'hello' - _OP, ':' - _STRING, 'greet=ing' - _OP, '=' - _STRING, 'world' - """ - current = '' - ops = ':=' - nextOps = {':': ':=', '=': ':'} - description = iter(description) - for n in description: - if n in ops: - yield _STRING, current - yield _OP, n - current = '' - ops = nextOps[n] - elif n == '\\': - current += next(description) - else: - current += n - yield _STRING, current - - - -def _parse(description): - """ - Convert a description string into a list of positional and keyword - parameters, using logic vaguely like what Python does. - - @param description: a string as described by L{serverFromString} or - L{clientFromString}. - - @return: a 2-tuple of C{(args, kwargs)}, where 'args' is a list of all - ':'-separated C{str}s not containing an '=' and 'kwargs' is a map of - all C{str}s which do contain an '='. For example, the result of - C{_parse('a:b:d=1:c')} would be C{(['a', 'b', 'c'], {'d': '1'})}. - """ - args, kw = [], {} - def add(sofar): - if len(sofar) == 1: - args.append(sofar[0]) - else: - kw[sofar[0]] = sofar[1] - sofar = () - for (type, value) in _tokenize(description): - if type is _STRING: - sofar += (value,) - elif value == ':': - add(sofar) - sofar = () - add(sofar) - return args, kw - - -# Mappings from description "names" to endpoint constructors. -_endpointServerFactories = { - 'TCP': TCP4ServerEndpoint, - 'SSL': SSL4ServerEndpoint, - 'UNIX': UNIXServerEndpoint, - } - -_endpointClientFactories = { - 'TCP': TCP4ClientEndpoint, - 'SSL': SSL4ClientEndpoint, - 'UNIX': UNIXClientEndpoint, - } - - -_NO_DEFAULT = object() - -def _parseServer(description, factory, default=None): - """ - Parse a stports description into a 2-tuple of arguments and keyword values. - - @param description: A description in the format explained by - L{serverFromString}. - @type description: C{str} - - @param factory: A 'factory' argument; this is left-over from - twisted.application.strports, it's not really used. - @type factory: L{IProtocolFactory} or L{None} - - @param default: Deprecated argument, specifying the default parser mode to - use for unqualified description strings (those which do not have a ':' - and prefix). - @type default: C{str} or C{NoneType} - - @return: a 3-tuple of (plugin or name, arguments, keyword arguments) - """ - args, kw = _parse(description) - if not args or (len(args) == 1 and not kw): - deprecationMessage = ( - "Unqualified strport description passed to 'service'." - "Use qualified endpoint descriptions; for example, 'tcp:%s'." - % (description,)) - if default is None: - default = 'tcp' - warnings.warn( - deprecationMessage, category=DeprecationWarning, stacklevel=4) - elif default is _NO_DEFAULT: - raise ValueError(deprecationMessage) - # If the default has been otherwise specified, the user has already - # been warned. - args[0:0] = [default] - endpointType = args[0] - parser = _serverParsers.get(endpointType) - if parser is None: - # If the required parser is not found in _server, check if - # a plugin exists for the endpointType - for plugin in getPlugins(IStreamServerEndpointStringParser): - if plugin.prefix == endpointType: - return (plugin, args[1:], kw) - raise ValueError("Unknown endpoint type: '%s'" % (endpointType,)) - return (endpointType.upper(),) + parser(factory, *args[1:], **kw) - - - -def _serverFromStringLegacy(reactor, description, default): - """ - Underlying implementation of L{serverFromString} which avoids exposing the - deprecated 'default' argument to anything but L{strports.service}. - """ - nameOrPlugin, args, kw = _parseServer(description, None, default) - if type(nameOrPlugin) is not str: - plugin = nameOrPlugin - return plugin.parseStreamServer(reactor, *args, **kw) - else: - name = nameOrPlugin - # Chop out the factory. - args = args[:1] + args[2:] - return _endpointServerFactories[name](reactor, *args, **kw) - - - -def serverFromString(reactor, description): - """ - Construct a stream server endpoint from an endpoint description string. - - The format for server endpoint descriptions is a simple string. It is a - prefix naming the type of endpoint, then a colon, then the arguments for - that endpoint. - - For example, you can call it like this to create an endpoint that will - listen on TCP port 80:: - - serverFromString(reactor, "tcp:80") - - Additional arguments may be specified as keywords, separated with colons. - For example, you can specify the interface for a TCP server endpoint to - bind to like this:: - - serverFromString(reactor, "tcp:80:interface=127.0.0.1") - - SSL server endpoints may be specified with the 'ssl' prefix, and the - private key and certificate files may be specified by the C{privateKey} and - C{certKey} arguments:: - - serverFromString(reactor, "ssl:443:privateKey=key.pem:certKey=crt.pem") - - If a private key file name (C{privateKey}) isn't provided, a "server.pem" - file is assumed to exist which contains the private key. If the certificate - file name (C{certKey}) isn't provided, the private key file is assumed to - contain the certificate as well. - - You may escape colons in arguments with a backslash, which you will need to - use if you want to specify a full pathname argument on Windows:: - - serverFromString(reactor, - "ssl:443:privateKey=C\\:/key.pem:certKey=C\\:/cert.pem") - - finally, the 'unix' prefix may be used to specify a filesystem UNIX socket, - optionally with a 'mode' argument to specify the mode of the socket file - created by C{listen}:: - - serverFromString(reactor, "unix:/var/run/finger") - serverFromString(reactor, "unix:/var/run/finger:mode=660") - - This function is also extensible; new endpoint types may be registered as - L{IStreamServerEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The server endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to listen with the parameters - given by by C{description}. - - @rtype: L{IStreamServerEndpoint} - - @raise ValueError: when the 'description' string cannot be parsed. - - @since: 10.2 - """ - return _serverFromStringLegacy(reactor, description, _NO_DEFAULT) - - - -def quoteStringArgument(argument): - """ - Quote an argument to L{serverFromString} and L{clientFromString}. Since - arguments are separated with colons and colons are escaped with - backslashes, some care is necessary if, for example, you have a pathname, - you may be tempted to interpolate into a string like this:: - - serverFromString("ssl:443:privateKey=%s" % (myPathName,)) - - This may appear to work, but will have portability issues (Windows - pathnames, for example). Usually you should just construct the appropriate - endpoint type rather than interpolating strings, which in this case would - be L{SSL4ServerEndpoint}. There are some use-cases where you may need to - generate such a string, though; for example, a tool to manipulate a - configuration file which has strports descriptions in it. To be correct in - those cases, do this instead:: - - serverFromString("ssl:443:privateKey=%s" % - (quoteStringArgument(myPathName),)) - - @param argument: The part of the endpoint description string you want to - pass through. - - @type argument: C{str} - - @return: The quoted argument. - - @rtype: C{str} - """ - return argument.replace('\\', '\\\\').replace(':', '\\:') - - - -def _parseClientTCP(*args, **kwargs): - """ - Perform any argument value coercion necessary for TCP client parameters. - - Valid positional arguments to this function are host and port. - - Valid keyword arguments to this function are all L{IReactorTCP.connectTCP} - arguments. - - @return: The coerced values as a C{dict}. - """ - - if len(args) == 2: - kwargs['port'] = int(args[1]) - kwargs['host'] = args[0] - elif len(args) == 1: - if 'host' in kwargs: - kwargs['port'] = int(args[0]) - else: - kwargs['host'] = args[0] - - try: - kwargs['port'] = int(kwargs['port']) - except KeyError: - pass - - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - - - -def _loadCAsFromDir(directoryPath): - """ - Load certificate-authority certificate objects in a given directory. - - @param directoryPath: a L{FilePath} pointing at a directory to load .pem - files from. - - @return: a C{list} of L{OpenSSL.crypto.X509} objects. - """ - from twisted.internet import ssl - - caCerts = {} - for child in directoryPath.children(): - if not child.basename().split('.')[-1].lower() == 'pem': - continue - try: - data = child.getContent() - except IOError: - # Permission denied, corrupt disk, we don't care. - continue - try: - theCert = ssl.Certificate.loadPEM(data) - except ssl.SSL.Error: - # Duplicate certificate, invalid certificate, etc. We don't care. - pass - else: - caCerts[theCert.digest()] = theCert.original - return caCerts.values() - - - -def _parseClientSSL(*args, **kwargs): - """ - Perform any argument value coercion necessary for SSL client parameters. - - Valid keyword arguments to this function are all L{IReactorSSL.connectSSL} - arguments except for C{contextFactory}. Instead, C{certKey} (the path name - of the certificate file) C{privateKey} (the path name of the private key - associated with the certificate) are accepted and used to construct a - context factory. - - Valid positional arguments to this function are host and port. - - @param caCertsDir: The one parameter which is not part of - L{IReactorSSL.connectSSL}'s signature, this is a path name used to - construct a list of certificate authority certificates. The directory - will be scanned for files ending in C{.pem}, all of which will be - considered valid certificate authorities for this connection. - - @type caCertsDir: C{str} - - @return: The coerced values as a C{dict}. - """ - from twisted.internet import ssl - kwargs = _parseClientTCP(*args, **kwargs) - certKey = kwargs.pop('certKey', None) - privateKey = kwargs.pop('privateKey', None) - caCertsDir = kwargs.pop('caCertsDir', None) - if certKey is not None: - certx509 = ssl.Certificate.loadPEM( - FilePath(certKey).getContent()).original - else: - certx509 = None - if privateKey is not None: - privateKey = ssl.PrivateCertificate.loadPEM( - FilePath(privateKey).getContent()).privateKey.original - else: - privateKey = None - if caCertsDir is not None: - verify = True - caCerts = _loadCAsFromDir(FilePath(caCertsDir)) - else: - verify = False - caCerts = None - kwargs['sslContextFactory'] = ssl.CertificateOptions( - method=ssl.SSL.SSLv23_METHOD, - certificate=certx509, - privateKey=privateKey, - verify=verify, - caCerts=caCerts - ) - return kwargs - - - -def _parseClientUNIX(*args, **kwargs): - """ - Perform any argument value coercion necessary for UNIX client parameters. - - Valid keyword arguments to this function are all L{IReactorUNIX.connectUNIX} - keyword arguments except for C{checkPID}. Instead, C{lockfile} is accepted - and has the same meaning. Also C{path} is used instead of C{address}. - - Valid positional arguments to this function are C{path}. - - @return: The coerced values as a C{dict}. - """ - if len(args) == 1: - kwargs['path'] = args[0] - - try: - kwargs['checkPID'] = bool(int(kwargs.pop('lockfile'))) - except KeyError: - pass - try: - kwargs['timeout'] = int(kwargs['timeout']) - except KeyError: - pass - return kwargs - -_clientParsers = { - 'TCP': _parseClientTCP, - 'SSL': _parseClientSSL, - 'UNIX': _parseClientUNIX, - } - - - -def clientFromString(reactor, description): - """ - Construct a client endpoint from a description string. - - Client description strings are much like server description strings, - although they take all of their arguments as keywords, aside from host and - port. - - You can create a TCP client endpoint with the 'host' and 'port' arguments, - like so:: - - clientFromString(reactor, "tcp:host=www.example.com:port=80") - - or, without specifying host and port keywords:: - - clientFromString(reactor, "tcp:www.example.com:80") - - Or you can specify only one or the other, as in the following 2 examples:: - - clientFromString(reactor, "tcp:host=www.example.com:80") - clientFromString(reactor, "tcp:www.example.com:port=80") - - or an SSL client endpoint with those arguments, plus the arguments used by - the server SSL, for a client certificate:: - - clientFromString(reactor, "ssl:web.example.com:443:" - "privateKey=foo.pem:certKey=foo.pem") - - to specify your certificate trust roots, you can identify a directory with - PEM files in it with the C{caCertsDir} argument:: - - clientFromString(reactor, "ssl:host=web.example.com:port=443:" - "caCertsDir=/etc/ssl/certs") - - You can create a UNIX client endpoint with the 'path' argument and optional - 'lockfile' and 'timeout' arguments:: - - clientFromString(reactor, "unix:path=/var/foo/bar:lockfile=1:timeout=9") - - or, with the path as a positional argument with or without optional - arguments as in the following 2 examples:: - - clientFromString(reactor, "unix:/var/foo/bar") - clientFromString(reactor, "unix:/var/foo/bar:lockfile=1:timeout=9") - - This function is also extensible; new endpoint types may be registered as - L{IStreamClientEndpointStringParser} plugins. See that interface for more - information. - - @param reactor: The client endpoint will be constructed with this reactor. - - @param description: The strports description to parse. - - @return: A new endpoint which can be used to connect with the parameters - given by by C{description}. - @rtype: L{IStreamClientEndpoint} - - @since: 10.2 - """ - args, kwargs = _parse(description) - aname = args.pop(0) - name = aname.upper() - for plugin in getPlugins(IStreamClientEndpointStringParser): - if plugin.prefix.upper() == name: - return plugin.parseStreamClient(*args, **kwargs) - if name not in _clientParsers: - raise ValueError("Unknown endpoint type: %r" % (aname,)) - kwargs = _clientParsers[name](*args, **kwargs) - return _endpointClientFactories[name](reactor, **kwargs) - - - -def connectProtocol(endpoint, protocol): - """ - Connect a protocol instance to an endpoint. - - This allows using a client endpoint without having to create a factory. - - @param endpoint: A client endpoint to connect to. - - @param protocol: A protocol instance. - - @return: The result of calling C{connect} on the endpoint, i.e. a - L{Deferred} that will fire with the protocol when connected, or an - appropriate error. - """ - class OneShotFactory(Factory): - def buildProtocol(self, addr): - return protocol - return endpoint.connect(OneShotFactory()) - diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py deleted file mode 100644 index f3e4ed5d8..000000000 --- a/scrapy/xlib/tx/interfaces.py +++ /dev/null @@ -1,2442 +0,0 @@ -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface documentation. - -Maintainer: Itamar Shtull-Trauring -""" - -from __future__ import division, absolute_import - -from zope.interface import Interface, Attribute - - -class IAddress(Interface): - """ - An address, e.g. a TCP C{(host, port)}. - - Default implementations are in L{twisted.internet.address}. - """ - -### Reactor Interfaces - -class IConnector(Interface): - """ - Object used to interface between connections and protocols. - - Each L{IConnector} manages one connection. - """ - - def stopConnecting(): - """ - Stop attempting to connect. - """ - - def disconnect(): - """ - Disconnect regardless of the connection state. - - If we are connected, disconnect, if we are trying to connect, - stop trying. - """ - - def connect(): - """ - Try to connect to remote address. - """ - - def getDestination(): - """ - Return destination this will try to connect to. - - @return: An object which provides L{IAddress}. - """ - - - -class IResolverSimple(Interface): - def getHostByName(name, timeout = (1, 3, 11, 45)): - """ - Resolve the domain name C{name} into an IP address. - - @type name: C{str} - @type timeout: C{tuple} - @rtype: L{twisted.internet.defer.Deferred} - @return: The callback of the Deferred that is returned will be - passed a string that represents the IP address of the specified - name, or the errback will be called if the lookup times out. If - multiple types of address records are associated with the name, - A6 records will be returned in preference to AAAA records, which - will be returned in preference to A records. If there are multiple - records of the type to be returned, one will be selected at random. - - @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) - if the name cannot be resolved within the specified timeout period. - """ - - - -class IResolver(IResolverSimple): - def query(query, timeout=None): - """ - Dispatch C{query} to the method which can handle its type. - - @type query: L{twisted.names.dns.Query} - @param query: The DNS query being issued, to which a response is to be - generated. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress(name, timeout=None): - """ - Perform an A record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAddress6(name, timeout=None): - """ - Perform an A6 record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupIPV6Address(name, timeout=None): - """ - Perform an AAAA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailExchange(name, timeout=None): - """ - Perform an MX record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNameservers(name, timeout=None): - """ - Perform an NS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupCanonicalName(name, timeout=None): - """ - Perform a CNAME record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailBox(name, timeout=None): - """ - Perform an MB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailGroup(name, timeout=None): - """ - Perform an MG record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailRename(name, timeout=None): - """ - Perform an MR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupPointer(name, timeout=None): - """ - Perform a PTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAuthority(name, timeout=None): - """ - Perform an SOA record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNull(name, timeout=None): - """ - Perform a NULL record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupWellKnownServices(name, timeout=None): - """ - Perform a WKS record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupHostInfo(name, timeout=None): - """ - Perform a HINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupMailboxInfo(name, timeout=None): - """ - Perform an MINFO record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupText(name, timeout=None): - """ - Perform a TXT record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupResponsibility(name, timeout=None): - """ - Perform an RP record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAFSDatabase(name, timeout=None): - """ - Perform an AFSDB record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupService(name, timeout=None): - """ - Perform an SRV record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupAllRecords(name, timeout=None): - """ - Perform an ALL_RECORD lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupSenderPolicy(name, timeout= 10): - """ - Perform a SPF record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupNamingAuthorityPointer(name, timeout=None): - """ - Perform a NAPTR record lookup. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: Sequence of C{int} - @param timeout: Number of seconds after which to reissue the query. - When the last timeout expires, the query is considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. The first element of the - tuple gives answers. The second element of the tuple gives - authorities. The third element of the tuple gives additional - information. The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - def lookupZone(name, timeout=None): - """ - Perform an AXFR record lookup. - - NB This is quite different from other DNS requests. See - U{http://cr.yp.to/djbdns/axfr-notes.html} for more - information. - - NB Unlike other C{lookup*} methods, the timeout here is not a - list of ints, it is a single int. - - @type name: C{str} - @param name: DNS name to resolve. - - @type timeout: C{int} - @param timeout: When this timeout expires, the query is - considered failed. - - @rtype: L{Deferred} - @return: A L{Deferred} which fires with a three-tuple of lists of - L{twisted.names.dns.RRHeader} instances. - The first element of the tuple gives answers. - The second and third elements are always empty. - The L{Deferred} may instead fail with one of the - exceptions defined in L{twisted.names.error} or with - C{NotImplementedError}. - """ - - - -class IReactorTCP(Interface): - - def listenTCP(port, factory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param backlog: size of the listen queue - - @param interface: The local IPv4 or IPv6 address to which to bind; - defaults to '', ie all IPv4 addresses. To bind to all IPv4 and IPv6 - addresses, you must call this method twice. - - @return: an object that provides L{IListeningPort}. - - @raise CannotListenError: as defined here - L{twisted.internet.error.CannotListenError}, - if it cannot listen on this port (e.g., it - cannot bind to the required port number) - """ - - def connectTCP(host, port, factory, timeout=30, bindAddress=None): - """ - Connect a TCP client. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind - to, or None. - - @return: An object which provides L{IConnector}. This connector will - call various callbacks on the factory when a connection is - made, failed, or lost - see - L{ClientFactory} - docs for details. - """ - -class IReactorSSL(Interface): - - def connectSSL(host, port, factory, contextFactory, timeout=30, bindAddress=None): - """ - Connect a client Protocol to a remote SSL socket. - - @param host: a host name - - @param port: a port number - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ClientContextFactory} object. - - @param timeout: number of seconds to wait before assuming the - connection has failed. - - @param bindAddress: a (host, port) tuple of local address to bind to, - or C{None}. - - @return: An object which provides L{IConnector}. - """ - - def listenSSL(port, factory, contextFactory, backlog=50, interface=''): - """ - Connects a given protocol factory to the given numeric TCP/IP port. - The connection is a SSL one, using contexts created by the context - factory. - - @param port: a port number on which to listen - - @param factory: a L{twisted.internet.protocol.ServerFactory} instance - - @param contextFactory: a L{twisted.internet.ssl.ContextFactory} instance - - @param backlog: size of the listen queue - - @param interface: the hostname to bind to, defaults to '' (all) - """ - - - -class IReactorUNIX(Interface): - """ - UNIX socket methods. - """ - - def connectUNIX(address, factory, timeout=30, checkPID=0): - """ - Connect a client protocol to a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.ClientFactory} instance - - @param timeout: number of seconds to wait before assuming the connection - has failed. - - @param checkPID: if True, check for a pid file to verify that a server - is listening. If C{address} is a Linux abstract namespace path, - this must be C{False}. - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIX(address, factory, backlog=50, mode=0o666, wantPID=0): - """ - Listen on a UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param factory: a L{twisted.internet.protocol.Factory} instance. - - @param backlog: number of connections to allow in backlog. - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param wantPID: if True, create a pidfile for the socket. If C{address} - is a Linux abstract namespace path, this must be C{False}. - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorUNIXDatagram(Interface): - """ - Datagram UNIX socket methods. - """ - - def connectUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666, bindAddress=None): - """ - Connect a client protocol to a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.ConnectedDatagramProtocol} instance - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @param bindAddress: address to bind to - - @return: An object which provides L{IConnector}. - """ - - - def listenUNIXDatagram(address, protocol, maxPacketSize=8192, mode=0o666): - """ - Listen on a datagram UNIX socket. - - @param address: a path to a unix socket on the filesystem. - - @param protocol: a L{twisted.internet.protocol.DatagramProtocol} instance. - - @param maxPacketSize: maximum packet size to accept - - @param mode: The mode (B{not} umask) to set on the unix socket. See - platform specific documentation for information about how this - might affect connection attempts. - @type mode: C{int} - - @return: An object which provides L{IListeningPort}. - """ - - - -class IReactorWin32Events(Interface): - """ - Win32 Event API methods - - @since: 10.2 - """ - - def addEvent(event, fd, action): - """ - Add a new win32 event to the event loop. - - @param event: a Win32 event object created using win32event.CreateEvent() - - @param fd: an instance of L{twisted.internet.abstract.FileDescriptor} - - @param action: a string that is a method name of the fd instance. - This method is called in response to the event. - - @return: None - """ - - - def removeEvent(event): - """ - Remove an event. - - @param event: a Win32 event object added using L{IReactorWin32Events.addEvent} - - @return: None - """ - - - -class IReactorUDP(Interface): - """ - UDP socket methods. - """ - - def listenUDP(port, protocol, interface='', maxPacketSize=8192): - """ - Connects a given DatagramProtocol to the given numeric UDP port. - - @return: object which provides L{IListeningPort}. - """ - - - -class IReactorMulticast(Interface): - """ - UDP socket methods that support multicast. - - IMPORTANT: This is an experimental new interface. It may change - without backwards compatibility. Suggestions are welcome. - """ - - def listenMulticast(port, protocol, interface='', maxPacketSize=8192, - listenMultiple=False): - """ - Connects a given - L{DatagramProtocol} to the - given numeric UDP port. - - @param listenMultiple: If set to True, allows multiple sockets to - bind to the same address and port number at the same time. - @type listenMultiple: C{bool} - - @returns: An object which provides L{IListeningPort}. - - @see: L{twisted.internet.interfaces.IMulticastTransport} - @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} - """ - - - -class IReactorSocket(Interface): - """ - Methods which allow a reactor to use externally created sockets. - - For example, to use C{adoptStreamPort} to implement behavior equivalent - to that of L{IReactorTCP.listenTCP}, you might write code like this:: - - from socket import SOMAXCONN, AF_INET, SOCK_STREAM, socket - portSocket = socket(AF_INET, SOCK_STREAM) - # Set FD_CLOEXEC on port, left as an exercise. Then make it into a - # non-blocking listening port: - portSocket.setblocking(False) - portSocket.bind(('192.168.1.2', 12345)) - portSocket.listen(SOMAXCONN) - - # Now have the reactor use it as a TCP port - port = reactor.adoptStreamPort( - portSocket.fileno(), AF_INET, YourFactory()) - - # portSocket itself is no longer necessary, and needs to be cleaned - # up by us. - portSocket.close() - - # Whenever the server is no longer needed, stop it as usual. - stoppedDeferred = port.stopListening() - - Another potential use is to inherit a listening descriptor from a parent - process (for example, systemd or launchd), or to receive one over a UNIX - domain socket. - - Some plans for extending this interface exist. See: - - - U{http://twistedmatrix.com/trac/ticket/5570}: established connections - - U{http://twistedmatrix.com/trac/ticket/5573}: AF_UNIX ports - - U{http://twistedmatrix.com/trac/ticket/5574}: SOCK_DGRAM sockets - """ - - def adoptStreamPort(fileDescriptor, addressFamily, factory): - """ - Add an existing listening I{SOCK_STREAM} socket to the reactor to - monitor for new connections to accept and handle. - - @param fileDescriptor: A file descriptor associated with a socket which - is already bound to an address and marked as listening. The socket - must be set non-blocking. Any additional flags (for example, - close-on-exec) must also be set by application code. Application - code is responsible for closing the file descriptor, which may be - done as soon as C{adoptStreamPort} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create new - protocols to handle connections accepted via this socket. - - @return: An object providing L{IListeningPort}. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - def adoptStreamConnection(fileDescriptor, addressFamily, factory): - """ - Add an existing connected I{SOCK_STREAM} socket to the reactor to - monitor for data. - - Note that the given factory won't have its C{startFactory} and - C{stopFactory} methods called, as there is no sensible time to call - them in this situation. - - @param fileDescriptor: A file descriptor associated with a socket which - is already connected. The socket must be set non-blocking. Any - additional flags (for example, close-on-exec) must also be set by - application code. Application code is responsible for closing the - file descriptor, which may be done as soon as - C{adoptStreamConnection} returns. - @type fileDescriptor: C{int} - - @param addressFamily: The address family (or I{domain}) of the socket. - For example, L{socket.AF_INET6}. - - @param factory: A L{ServerFactory} instance to use to create a new - protocol to handle the connection via this socket. - - @raise UnsupportedAddressFamily: If the given address family is not - supported by this reactor, or not supported with the given socket - type. - - @raise UnsupportedSocketType: If the given socket type is not supported - by this reactor, or not supported with the given socket type. - """ - - - -class IReactorProcess(Interface): - - def spawnProcess(processProtocol, executable, args=(), env={}, path=None, - uid=None, gid=None, usePTY=0, childFDs=None): - """ - Spawn a process, with a process protocol. - - @type processProtocol: L{IProcessProtocol} provider - @param processProtocol: An object which will be notified of all - events related to the created process. - - @param executable: the file name to spawn - the full path should be - used. - - @param args: the command line arguments to pass to the process; a - sequence of strings. The first string should be the - executable's name. - - @type env: a C{dict} mapping C{str} to C{str}, or C{None}. - @param env: the environment variables to pass to the child process. The - resulting behavior varies between platforms. If - - C{env} is not set: - - On POSIX: pass an empty environment. - - On Windows: pass C{os.environ}. - - C{env} is C{None}: - - On POSIX: pass C{os.environ}. - - On Windows: pass C{os.environ}. - - C{env} is a C{dict}: - - On POSIX: pass the key/value pairs in C{env} as the - complete environment. - - On Windows: update C{os.environ} with the key/value - pairs in the C{dict} before passing it. As a - consequence of U{bug #1640 - }, passing - keys with empty values in an effort to unset - environment variables I{won't} unset them. - - @param path: the path to run the subprocess in - defaults to the - current directory. - - @param uid: user ID to run the subprocess as. (Only available on - POSIX systems.) - - @param gid: group ID to run the subprocess as. (Only available on - POSIX systems.) - - @param usePTY: if true, run this process in a pseudo-terminal. - optionally a tuple of C{(masterfd, slavefd, ttyname)}, - in which case use those file descriptors. - (Not available on all systems.) - - @param childFDs: A dictionary mapping file descriptors in the new child - process to an integer or to the string 'r' or 'w'. - - If the value is an integer, it specifies a file - descriptor in the parent process which will be mapped - to a file descriptor (specified by the key) in the - child process. This is useful for things like inetd - and shell-like file redirection. - - If it is the string 'r', a pipe will be created and - attached to the child at that file descriptor: the - child will be able to write to that file descriptor - and the parent will receive read notification via the - L{IProcessProtocol.childDataReceived} callback. This - is useful for the child's stdout and stderr. - - If it is the string 'w', similar setup to the previous - case will occur, with the pipe being readable by the - child instead of writeable. The parent process can - write to that file descriptor using - L{IProcessTransport.writeToChild}. This is useful for - the child's stdin. - - If childFDs is not passed, the default behaviour is to - use a mapping that opens the usual stdin/stdout/stderr - pipes. - - @see: L{twisted.internet.protocol.ProcessProtocol} - - @return: An object which provides L{IProcessTransport}. - - @raise OSError: Raised with errno C{EAGAIN} or C{ENOMEM} if there are - insufficient system resources to create a new process. - """ - -class IReactorTime(Interface): - """ - Time methods that a Reactor should implement. - """ - - def seconds(): - """ - Get the current time in seconds. - - @return: A number-like object of some sort. - """ - - - def callLater(delay, callable, *args, **kw): - """ - Call a function later. - - @type delay: C{float} - @param delay: the number of seconds to wait. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: An object which provides L{IDelayedCall} and can be used to - cancel the scheduled call, by calling its C{cancel()} method. - It also may be rescheduled by calling its C{delay()} or - C{reset()} methods. - """ - - - def getDelayedCalls(): - """ - Retrieve all currently scheduled delayed calls. - - @return: A tuple of all L{IDelayedCall} providers representing all - currently scheduled calls. This is everything that has been - returned by C{callLater} but not yet called or canceled. - """ - - -class IDelayedCall(Interface): - """ - A scheduled call. - - There are probably other useful methods we can add to this interface; - suggestions are welcome. - """ - - def getTime(): - """ - Get time when delayed call will happen. - - @return: time in seconds since epoch (a float). - """ - - def cancel(): - """ - Cancel the scheduled call. - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def delay(secondsLater): - """ - Delay the scheduled call. - - @param secondsLater: how many seconds from its current firing time to delay - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def reset(secondsFromNow): - """ - Reset the scheduled call's timer. - - @param secondsFromNow: how many seconds from now it should fire, - equivalent to C{.cancel()} and then doing another - C{reactor.callLater(secondsLater, ...)} - - @raises twisted.internet.error.AlreadyCalled: if the call has already - happened. - @raises twisted.internet.error.AlreadyCancelled: if the call has already - been cancelled. - """ - - def active(): - """ - @return: True if this call is still active, False if it has been - called or cancelled. - """ - -class IReactorThreads(Interface): - """ - Dispatch methods to be run in threads. - - Internally, this should use a thread pool and dispatch methods to them. - """ - - def getThreadPool(): - """ - Return the threadpool used by L{callInThread}. Create it first if - necessary. - - @rtype: L{twisted.python.threadpool.ThreadPool} - """ - - - def callInThread(callable, *args, **kwargs): - """ - Run the callable object in a separate thread. - """ - - - def callFromThread(callable, *args, **kw): - """ - Cause a function to be executed by the reactor thread. - - Use this method when you want to run a function in the reactor's thread - from another thread. Calling L{callFromThread} should wake up the main - thread (where L{reactor.run()} is executing) and run the - given callable in that thread. - - If you're writing a multi-threaded application the C{callable} may need - to be thread safe, but this method doesn't require it as such. If you - want to call a function in the next mainloop iteration, but you're in - the same thread, use L{callLater} with a delay of 0. - """ - - - def suggestThreadPoolSize(size): - """ - Suggest the size of the internal threadpool used to dispatch functions - passed to L{callInThread}. - """ - - -class IReactorCore(Interface): - """ - Core methods that a Reactor must implement. - """ - - running = Attribute( - "A C{bool} which is C{True} from I{during startup} to " - "I{during shutdown} and C{False} the rest of the time.") - - - def resolve(name, timeout=10): - """ - Return a L{twisted.internet.defer.Deferred} that will resolve a hostname. - """ - - def run(): - """ - Fire 'startup' System Events, move the reactor to the 'running' - state, then run the main loop until it is stopped with C{stop()} or - C{crash()}. - """ - - def stop(): - """ - Fire 'shutdown' System Events, which will move the reactor to the - 'stopped' state and cause C{reactor.run()} to exit. - """ - - def crash(): - """ - Stop the main loop *immediately*, without firing any system events. - - This is named as it is because this is an extremely "rude" thing to do; - it is possible to lose data and put your system in an inconsistent - state by calling this. However, it is necessary, as sometimes a system - can become wedged in a pre-shutdown call. - """ - - def iterate(delay=0): - """ - Run the main loop's I/O polling function for a period of time. - - This is most useful in applications where the UI is being drawn "as - fast as possible", such as games. All pending L{IDelayedCall}s will - be called. - - The reactor must have been started (via the C{run()} method) prior to - any invocations of this method. It must also be stopped manually - after the last call to this method (via the C{stop()} method). This - method is not re-entrant: you must not call it recursively; in - particular, you must not call it while the reactor is running. - """ - - def fireSystemEvent(eventType): - """ - Fire a system-wide event. - - System-wide events are things like 'startup', 'shutdown', and - 'persist'. - """ - - def addSystemEventTrigger(phase, eventType, callable, *args, **kw): - """ - Add a function to be called when a system event occurs. - - Each "system event" in Twisted, such as 'startup', 'shutdown', and - 'persist', has 3 phases: 'before', 'during', and 'after' (in that - order, of course). These events will be fired internally by the - Reactor. - - An implementor of this interface must only implement those events - described here. - - Callbacks registered for the "before" phase may return either None or a - Deferred. The "during" phase will not execute until all of the - Deferreds from the "before" phase have fired. - - Once the "during" phase is running, all of the remaining triggers must - execute; their return values must be ignored. - - @param phase: a time to call the event -- either the string 'before', - 'after', or 'during', describing when to call it - relative to the event's execution. - - @param eventType: this is a string describing the type of event. - - @param callable: the object to call before shutdown. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: an ID that can be used to remove this call with - removeSystemEventTrigger. - """ - - def removeSystemEventTrigger(triggerID): - """ - Removes a trigger added with addSystemEventTrigger. - - @param triggerID: a value returned from addSystemEventTrigger. - - @raise KeyError: If there is no system event trigger for the given - C{triggerID}. - - @raise ValueError: If there is no system event trigger for the given - C{triggerID}. - - @raise TypeError: If there is no system event trigger for the given - C{triggerID}. - """ - - def callWhenRunning(callable, *args, **kw): - """ - Call a function when the reactor is running. - - If the reactor has not started, the callable will be scheduled - to run when it does start. Otherwise, the callable will be invoked - immediately. - - @param callable: the callable object to call later. - - @param args: the arguments to call it with. - - @param kw: the keyword arguments to call it with. - - @return: None if the callable was invoked, otherwise a system - event id for the scheduled call. - """ - - -class IReactorPluggableResolver(Interface): - """ - A reactor with a pluggable name resolver interface. - """ - - def installResolver(resolver): - """ - Set the internal resolver to use to for name lookups. - - @type resolver: An object implementing the L{IResolverSimple} interface - @param resolver: The new resolver to use. - - @return: The previously installed resolver. - """ - - -class IReactorDaemonize(Interface): - """ - A reactor which provides hooks that need to be called before and after - daemonization. - - Notes: - - This interface SHOULD NOT be called by applications. - - This interface should only be implemented by reactors as a workaround - (in particular, it's implemented currently only by kqueue()). - For details please see the comments on ticket #1918. - """ - - def beforeDaemonize(): - """ - Hook to be called immediately before daemonization. No reactor methods - may be called until L{afterDaemonize} is called. - - @return: C{None}. - """ - - - def afterDaemonize(): - """ - Hook to be called immediately after daemonization. This may only be - called after L{beforeDaemonize} had been called previously. - - @return: C{None}. - """ - - - -class IReactorFDSet(Interface): - """ - Implement me to be able to use L{IFileDescriptor} type resources. - - This assumes that your main-loop uses UNIX-style numeric file descriptors - (or at least similarly opaque IDs returned from a .fileno() method) - """ - - def addReader(reader): - """ - I add reader to the set of file descriptors to get read events for. - - @param reader: An L{IReadDescriptor} provider that will be checked for - read events until it is removed from the reactor with - L{removeReader}. - - @return: C{None}. - """ - - def addWriter(writer): - """ - I add writer to the set of file descriptors to get write events for. - - @param writer: An L{IWriteDescriptor} provider that will be checked for - write events until it is removed from the reactor with - L{removeWriter}. - - @return: C{None}. - """ - - def removeReader(reader): - """ - Removes an object previously added with L{addReader}. - - @return: C{None}. - """ - - def removeWriter(writer): - """ - Removes an object previously added with L{addWriter}. - - @return: C{None}. - """ - - def removeAll(): - """ - Remove all readers and writers. - - Should not remove reactor internal reactor connections (like a waker). - - @return: A list of L{IReadDescriptor} and L{IWriteDescriptor} providers - which were removed. - """ - - def getReaders(): - """ - Return the list of file descriptors currently monitored for input - events by the reactor. - - @return: the list of file descriptors monitored for input events. - @rtype: C{list} of C{IReadDescriptor} - """ - - def getWriters(): - """ - Return the list file descriptors currently monitored for output events - by the reactor. - - @return: the list of file descriptors monitored for output events. - @rtype: C{list} of C{IWriteDescriptor} - """ - - -class IListeningPort(Interface): - """ - A listening port. - """ - - def startListening(): - """ - Start listening on this port. - - @raise CannotListenError: If it cannot listen on this port (e.g., it is - a TCP port and it cannot bind to the required - port number). - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return Deferred that fires - upon completion. - """ - - def getHost(): - """ - Get the host that this port is listening for. - - @return: An L{IAddress} provider. - """ - - -class ILoggingContext(Interface): - """ - Give context information that will be used to log events generated by - this item. - """ - - def logPrefix(): - """ - @return: Prefix used during log formatting to indicate context. - @rtype: C{str} - """ - - - -class IFileDescriptor(ILoggingContext): - """ - An interface representing a UNIX-style numeric file descriptor. - """ - - def fileno(): - """ - @raise: If the descriptor no longer has a valid file descriptor - number associated with it. - - @return: The platform-specified representation of a file descriptor - number. Or C{-1} if the descriptor no longer has a valid file - descriptor number associated with it. As long as the descriptor - is valid, calls to this method on a particular instance must - return the same value. - """ - - - def connectionLost(reason): - """ - Called when the connection was lost. - - This is called when the connection on a selectable object has been - lost. It will be called whether the connection was closed explicitly, - an exception occurred in an event handler, or the other end of the - connection closed it first. - - See also L{IHalfCloseableDescriptor} if your descriptor wants to be - notified separately of the two halves of the connection being closed. - - @param reason: A failure instance indicating the reason why the - connection was lost. L{error.ConnectionLost} and - L{error.ConnectionDone} are of special note, but the - failure may be of other classes as well. - """ - - - -class IReadDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can read. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doRead(): - """ - Some data is available for reading on your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IWriteDescriptor(IFileDescriptor): - """ - An L{IFileDescriptor} that can write. - - This interface is generally used in conjunction with L{IReactorFDSet}. - """ - - def doWrite(): - """ - Some data can be written to your descriptor. - - @return: If an error is encountered which causes the descriptor to - no longer be valid, a L{Failure} should be returned. Otherwise, - C{None}. - """ - - -class IReadWriteDescriptor(IReadDescriptor, IWriteDescriptor): - """ - An L{IFileDescriptor} that can both read and write. - """ - - -class IHalfCloseableDescriptor(Interface): - """ - A descriptor that can be half-closed. - """ - - def writeConnectionLost(reason): - """ - Indicates write connection was lost. - """ - - def readConnectionLost(reason): - """ - Indicates read connection was lost. - """ - - -class ISystemHandle(Interface): - """ - An object that wraps a networking OS-specific handle. - """ - - def getHandle(): - """ - Return a system- and reactor-specific handle. - - This might be a socket.socket() object, or some other type of - object, depending on which reactor is being used. Use and - manipulate at your own risk. - - This might be used in cases where you want to set specific - options not exposed by the Twisted APIs. - """ - - -class IConsumer(Interface): - """ - A consumer consumes data from a producer. - """ - - def registerProducer(producer, streaming): - """ - Register to receive data from a producer. - - This sets self to be a consumer for a producer. When this object runs - out of data (as when a send(2) call on a socket succeeds in moving the - last data from a userspace buffer into a kernelspace buffer), it will - ask the producer to resumeProducing(). - - For L{IPullProducer} providers, C{resumeProducing} will be called once - each time data is required. - - For L{IPushProducer} providers, C{pauseProducing} will be called - whenever the write buffer fills up and C{resumeProducing} will only be - called when it empties. - - @type producer: L{IProducer} provider - - @type streaming: C{bool} - @param streaming: C{True} if C{producer} provides L{IPushProducer}, - C{False} if C{producer} provides L{IPullProducer}. - - @raise RuntimeError: If a producer is already registered. - - @return: C{None} - """ - - - def unregisterProducer(): - """ - Stop consuming data from a producer, without disconnecting. - """ - - - def write(data): - """ - The producer will write data by calling this method. - - The implementation must be non-blocking and perform whatever - buffering is necessary. If the producer has provided enough data - for now and it is a L{IPushProducer}, the consumer may call its - C{pauseProducing} method. - """ - - - -class IProducer(Interface): - """ - A producer produces data for a consumer. - - Typically producing is done by calling the write method of an class - implementing L{IConsumer}. - """ - - def stopProducing(): - """ - Stop producing data. - - This tells a producer that its consumer has died, so it must stop - producing data for good. - """ - - -class IPushProducer(IProducer): - """ - A push producer, also known as a streaming producer is expected to - produce (write to this consumer) data on a continuous basis, unless - it has been paused. A paused push producer will resume producing - after its resumeProducing() method is called. For a push producer - which is not pauseable, these functions may be noops. - """ - - def pauseProducing(): - """ - Pause producing data. - - Tells a producer that it has produced too much data to process for - the time being, and to stop until resumeProducing() is called. - """ - def resumeProducing(): - """ - Resume producing data. - - This tells a producer to re-add itself to the main loop and produce - more data for its consumer. - """ - -class IPullProducer(IProducer): - """ - A pull producer, also known as a non-streaming producer, is - expected to produce data each time resumeProducing() is called. - """ - - def resumeProducing(): - """ - Produce data for the consumer a single time. - - This tells a producer to produce data for the consumer once - (not repeatedly, once only). Typically this will be done - by calling the consumer's write() method a single time with - produced data. - """ - -class IProtocol(Interface): - - def dataReceived(data): - """ - Called whenever data is received. - - Use this method to translate to a higher-level message. Usually, some - callback will be made upon the receipt of each complete protocol - message. - - @param data: a string of indeterminate length. Please keep in mind - that you will probably need to buffer some data, as partial - (or multiple) protocol messages may be received! I recommend - that unit tests for protocols call through to this method with - differing chunk sizes, down to one byte at a time. - """ - - def connectionLost(reason): - """ - Called when the connection is shut down. - - Clear any circular references here, and any external references - to this Protocol. The connection has been closed. The C{reason} - Failure wraps a L{twisted.internet.error.ConnectionDone} or - L{twisted.internet.error.ConnectionLost} instance (or a subclass - of one of those). - - @type reason: L{twisted.python.failure.Failure} - """ - - def makeConnection(transport): - """ - Make a connection to a transport and a server. - """ - - def connectionMade(): - """ - Called when a connection is made. - - This may be considered the initializer of the protocol, because - it is called when the connection is completed. For clients, - this is called once the connection to the server has been - established; for servers, this is called after an accept() call - stops blocking and a socket has been received. If you need to - send any greeting or initial message, do it here. - """ - - -class IProcessProtocol(Interface): - """ - Interface for process-related event handlers. - """ - - def makeConnection(process): - """ - Called when the process has been created. - - @type process: L{IProcessTransport} provider - @param process: An object representing the process which has been - created and associated with this protocol. - """ - - - def childDataReceived(childFD, data): - """ - Called when data arrives from the child process. - - @type childFD: C{int} - @param childFD: The file descriptor from which the data was - received. - - @type data: C{str} - @param data: The data read from the child's file descriptor. - """ - - - def childConnectionLost(childFD): - """ - Called when a file descriptor associated with the child process is - closed. - - @type childFD: C{int} - @param childFD: The file descriptor which was closed. - """ - - - def processExited(reason): - """ - Called when the child process exits. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - - @since: 8.2 - """ - - - def processEnded(reason): - """ - Called when the child process exits and all file descriptors associated - with it have been closed. - - @type reason: L{twisted.python.failure.Failure} - @param reason: A failure giving the reason the child process - terminated. The type of exception for this failure is either - L{twisted.internet.error.ProcessDone} or - L{twisted.internet.error.ProcessTerminated}. - """ - - - -class IHalfCloseableProtocol(Interface): - """ - Implemented to indicate they want notification of half-closes. - - TCP supports the notion of half-closing the connection, e.g. - closing the write side but still not stopping reading. A protocol - that implements this interface will be notified of such events, - instead of having connectionLost called. - """ - - def readConnectionLost(): - """ - Notification of the read connection being closed. - - This indicates peer did half-close of write side. It is now - the responsibility of the this protocol to call - loseConnection(). In addition, the protocol MUST make sure a - reference to it still exists (i.e. by doing a callLater with - one of its methods, etc.) as the reactor will only have a - reference to it if it is writing. - - If the protocol does not do so, it might get garbage collected - without the connectionLost method ever being called. - """ - - def writeConnectionLost(): - """ - Notification of the write connection being closed. - - This will never be called for TCP connections as TCP does not - support notification of this type of half-close. - """ - - - -class IFileDescriptorReceiver(Interface): - """ - Protocols may implement L{IFileDescriptorReceiver} to receive file - descriptors sent to them. This is useful in conjunction with - L{IUNIXTransport}, which allows file descriptors to be sent between - processes on a single host. - """ - def fileDescriptorReceived(descriptor): - """ - Called when a file descriptor is received over the connection. - - @param descriptor: The descriptor which was received. - @type descriptor: C{int} - - @return: C{None} - """ - - - -class IProtocolFactory(Interface): - """ - Interface for protocol factories. - """ - - def buildProtocol(addr): - """ - Called when a connection has been established to addr. - - If None is returned, the connection is assumed to have been refused, - and the Port will close the connection. - - @type addr: (host, port) - @param addr: The address of the newly-established connection - - @return: None if the connection was refused, otherwise an object - providing L{IProtocol}. - """ - - def doStart(): - """ - Called every time this is connected to a Port or Connector. - """ - - def doStop(): - """ - Called every time this is unconnected from a Port or Connector. - """ - - -class ITransport(Interface): - """ - I am a transport for bytes. - - I represent (and wrap) the physical connection and synchronicity - of the framework which is talking to the network. I make no - representations about whether calls to me will happen immediately - or require returning to a control loop, or whether they will happen - in the same or another thread. Consider methods of this class - (aside from getPeer) to be 'thrown over the wall', to happen at some - indeterminate time. - """ - - def write(data): - """ - Write some data to the physical connection, in sequence, in a - non-blocking fashion. - - If possible, make sure that it is all written. No data will - ever be lost, although (obviously) the connection may be closed - before it all gets through. - """ - - def writeSequence(data): - """ - Write a list of strings to the physical connection. - - If possible, make sure that all of the data is written to - the socket at once, without first copying it all into a - single string. - """ - - def loseConnection(): - """ - Close my connection, after writing all pending data. - - Note that if there is a registered producer on a transport it - will not be closed until the producer has been unregistered. - """ - - def getPeer(): - """ - Get the remote address of this connection. - - Treat this method with caution. It is the unfortunate result of the - CGI and Jabber standards, but should not be considered reliable for - the usual host of reasons; port forwarding, proxying, firewalls, IP - masquerading, etc. - - @return: An L{IAddress} provider. - """ - - def getHost(): - """ - Similar to getPeer, but returns an address describing this side of the - connection. - - @return: An L{IAddress} provider. - """ - - -class ITCPTransport(ITransport): - """ - A TCP based transport. - """ - - def loseWriteConnection(): - """ - Half-close the write side of a TCP connection. - - If the protocol instance this is attached to provides - IHalfCloseableProtocol, it will get notified when the operation is - done. When closing write connection, as with loseConnection this will - only happen when buffer has emptied and there is no registered - producer. - """ - - - def abortConnection(): - """ - Close the connection abruptly. - - Discards any buffered data, stops any registered producer, - and, if possible, notifies the other end of the unclean - closure. - - @since: 11.1 - """ - - - def getTcpNoDelay(): - """ - Return if C{TCP_NODELAY} is enabled. - """ - - def setTcpNoDelay(enabled): - """ - Enable/disable C{TCP_NODELAY}. - - Enabling C{TCP_NODELAY} turns off Nagle's algorithm. Small packets are - sent sooner, possibly at the expense of overall throughput. - """ - - def getTcpKeepAlive(): - """ - Return if C{SO_KEEPALIVE} is enabled. - """ - - def setTcpKeepAlive(enabled): - """ - Enable/disable C{SO_KEEPALIVE}. - - Enabling C{SO_KEEPALIVE} sends packets periodically when the connection - is otherwise idle, usually once every two hours. They are intended - to allow detection of lost peers in a non-infinite amount of time. - """ - - def getHost(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - def getPeer(): - """ - Returns L{IPv4Address} or L{IPv6Address}. - """ - - - -class IUNIXTransport(ITransport): - """ - Transport for stream-oriented unix domain connections. - """ - def sendFileDescriptor(descriptor): - """ - Send a duplicate of this (file, socket, pipe, etc) descriptor to the - other end of this connection. - - The send is non-blocking and will be queued if it cannot be performed - immediately. The send will be processed in order with respect to other - C{sendFileDescriptor} calls on this transport, but not necessarily with - respect to C{write} calls on this transport. The send can only be - processed if there are also bytes in the normal connection-oriented send - buffer (ie, you must call C{write} at least as many times as you call - C{sendFileDescriptor}). - - @param descriptor: An C{int} giving a valid file descriptor in this - process. Note that a I{file descriptor} may actually refer to a - socket, a pipe, or anything else POSIX tries to treat in the same - way as a file. - - @return: C{None} - """ - - - -class ITLSTransport(ITCPTransport): - """ - A TCP transport that supports switching to TLS midstream. - - Once TLS mode is started the transport will implement L{ISSLTransport}. - """ - - def startTLS(contextFactory): - """ - Initiate TLS negotiation. - - @param contextFactory: A context factory (see L{ssl.py}) - """ - -class ISSLTransport(ITCPTransport): - """ - A SSL/TLS based transport. - """ - - def getPeerCertificate(): - """ - Return an object with the peer's certificate info. - """ - - -class IProcessTransport(ITransport): - """ - A process transport. - """ - - pid = Attribute( - "From before L{IProcessProtocol.makeConnection} is called to before " - "L{IProcessProtocol.processEnded} is called, C{pid} is an L{int} " - "giving the platform process ID of this process. C{pid} is L{None} " - "at all other times.") - - def closeStdin(): - """ - Close stdin after all data has been written out. - """ - - def closeStdout(): - """ - Close stdout. - """ - - def closeStderr(): - """ - Close stderr. - """ - - def closeChildFD(descriptor): - """ - Close a file descriptor which is connected to the child process, identified - by its FD in the child process. - """ - - def writeToChild(childFD, data): - """ - Similar to L{ITransport.write} but also allows the file descriptor in - the child process which will receive the bytes to be specified. - - @type childFD: C{int} - @param childFD: The file descriptor to which to write. - - @type data: C{str} - @param data: The bytes to write. - - @return: C{None} - - @raise KeyError: If C{childFD} is not a file descriptor that was mapped - in the child when L{IReactorProcess.spawnProcess} was used to create - it. - """ - - def loseConnection(): - """ - Close stdin, stderr and stdout. - """ - - def signalProcess(signalID): - """ - Send a signal to the process. - - @param signalID: can be - - one of C{"KILL"}, C{"TERM"}, or C{"INT"}. - These will be implemented in a - cross-platform manner, and so should be used - if possible. - - an integer, where it represents a POSIX - signal ID. - - @raise twisted.internet.error.ProcessExitedAlready: If the process has - already exited. - @raise OSError: If the C{os.kill} call fails with an errno different - from C{ESRCH}. - """ - - -class IServiceCollection(Interface): - """ - An object which provides access to a collection of services. - """ - - def getServiceNamed(serviceName): - """ - Retrieve the named service from this application. - - Raise a C{KeyError} if there is no such service name. - """ - - def addService(service): - """ - Add a service to this collection. - """ - - def removeService(service): - """ - Remove a service from this collection. - """ - - -class IUDPTransport(Interface): - """ - Transport for UDP DatagramProtocols. - """ - - def write(packet, addr=None): - """ - Write packet to given address. - - @param addr: a tuple of (ip, port). For connected transports must - be the address the transport is connected to, or None. - In non-connected mode this is mandatory. - - @raise twisted.internet.error.MessageLengthError: C{packet} was too - long. - """ - - def connect(host, port): - """ - Connect the transport to an address. - - This changes it to connected mode. Datagrams can only be sent to - this address, and will only be received from this address. In addition - the protocol's connectionRefused method might get called if destination - is not receiving datagrams. - - @param host: an IP address, not a domain name ('127.0.0.1', not 'localhost') - @param port: port to connect to. - """ - - def getHost(): - """ - Returns L{IPv4Address}. - """ - - def stopListening(): - """ - Stop listening on this port. - - If it does not complete immediately, will return L{Deferred} that fires - upon completion. - """ - - - -class IUNIXDatagramTransport(Interface): - """ - Transport for UDP PacketProtocols. - """ - - def write(packet, address): - """ - Write packet to given address. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - -class IUNIXDatagramConnectedTransport(Interface): - """ - Transport for UDP ConnectedPacketProtocols. - """ - - def write(packet): - """ - Write packet to address we are connected to. - """ - - def getHost(): - """ - Returns L{UNIXAddress}. - """ - - def getPeer(): - """ - Returns L{UNIXAddress}. - """ - - -class IMulticastTransport(Interface): - """ - Additional functionality for multicast UDP. - """ - - def getOutgoingInterface(): - """ - Return interface of outgoing multicast packets. - """ - - def setOutgoingInterface(addr): - """ - Set interface for outgoing multicast packets. - - Returns Deferred of success. - """ - - def getLoopbackMode(): - """ - Return if loopback mode is enabled. - """ - - def setLoopbackMode(mode): - """ - Set if loopback mode is enabled. - """ - - def getTTL(): - """ - Get time to live for multicast packets. - """ - - def setTTL(ttl): - """ - Set time to live on multicast packets. - """ - - def joinGroup(addr, interface=""): - """ - Join a multicast group. Returns L{Deferred} of success or failure. - - If an error occurs, the returned L{Deferred} will fail with - L{error.MulticastJoinError}. - """ - - def leaveGroup(addr, interface=""): - """ - Leave multicast group, return L{Deferred} of success. - """ - - -class IStreamClientEndpoint(Interface): - """ - A stream client endpoint is a place that L{ClientFactory} can connect to. - For example, a remote TCP host/port pair would be a TCP client endpoint. - - @since: 10.1 - """ - - def connect(protocolFactory): - """ - Connect the C{protocolFactory} to the location specified by this - L{IStreamClientEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IProtocol} upon successful - connection otherwise a L{ConnectError} - """ - - - -class IStreamServerEndpoint(Interface): - """ - A stream server endpoint is a place that a L{Factory} can listen for - incoming connections. - - @since: 10.1 - """ - - def listen(protocolFactory): - """ - Listen with C{protocolFactory} at the location specified by this - L{IStreamServerEndpoint} provider. - - @param protocolFactory: A provider of L{IProtocolFactory} - @return: A L{Deferred} that results in an L{IListeningPort} or an - L{CannotListenError} - """ - - - -class IStreamServerEndpointStringParser(Interface): - """ - An L{IStreamServerEndpointStringParser} is like an - L{IStreamClientEndpointStringParser}, except for L{IStreamServerEndpoint}s - instead of clients. It integrates with L{endpoints.serverFromString} in - much the same way. - """ - - prefix = Attribute( - """ - @see: L{IStreamClientEndpointStringParser.prefix} - """ - ) - - - def parseStreamServer(reactor, *args, **kwargs): - """ - Parse a stream server endpoint from a reactor and string-only arguments - and keyword arguments. - - @see: L{IStreamClientEndpointStringParser.parseStreamClient} - - @return: a stream server endpoint - @rtype: L{IStreamServerEndpoint} - """ - - - -class IStreamClientEndpointStringParser(Interface): - """ - An L{IStreamClientEndpointStringParser} is a parser which can convert - a set of string C{*args} and C{**kwargs} into an L{IStreamClientEndpoint} - provider. - - This interface is really only useful in the context of the plugin system - for L{endpoints.clientFromString}. See the document entitled "I{The - Twisted Plugin System}" for more details on how to write a plugin. - - If you place an L{IStreamClientEndpointStringParser} plugin in the - C{twisted.plugins} package, that plugin's C{parseStreamClient} method will - be used to produce endpoints for any description string that begins with - the result of that L{IStreamClientEndpointStringParser}'s prefix attribute. - """ - - prefix = Attribute( - """ - A C{str}, the description prefix to respond to. For example, an - L{IStreamClientEndpointStringParser} plugin which had C{"foo"} for its - C{prefix} attribute would be called for endpoint descriptions like - C{"foo:bar:baz"} or C{"foo:"}. - """ - ) - - - def parseStreamClient(*args, **kwargs): - """ - This method is invoked by L{endpoints.clientFromString}, if the type of - endpoint matches the return value from this - L{IStreamClientEndpointStringParser}'s C{prefix} method. - - @param args: The string arguments, minus the endpoint type, in the - endpoint description string, parsed according to the rules - described in L{endpoints.quoteStringArgument}. For example, if the - description were C{"my-type:foo:bar:baz=qux"}, C{args} would be - C{('foo','bar')} - - @param kwargs: The string arguments from the endpoint description - passed as keyword arguments. For example, if the description were - C{"my-type:foo:bar:baz=qux"}, C{kwargs} would be - C{dict(baz='qux')}. - - @return: a client endpoint - @rtype: L{IStreamClientEndpoint} - """ diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py deleted file mode 100644 index ddcb6ed7a..000000000 --- a/scrapy/xlib/tx/iweb.py +++ /dev/null @@ -1,587 +0,0 @@ -# -*- test-case-name: twisted.web.test -*- -# Copyright (c) Twisted Matrix Laboratories. -# See LICENSE for details. - -""" -Interface definitions for L{twisted.web}. - -@var UNKNOWN_LENGTH: An opaque object which may be used as the value of - L{IBodyProducer.length} to indicate that the length of the entity - body is not known in advance. -""" - -from zope.interface import Interface, Attribute - -from twisted.internet.interfaces import IPushProducer - - -class IRequest(Interface): - """ - An HTTP request. - - @since: 9.0 - """ - - method = Attribute("A C{str} giving the HTTP method that was used.") - uri = Attribute( - "A C{str} giving the full encoded URI which was requested (including " - "query arguments).") - path = Attribute( - "A C{str} giving the encoded query path of the request URI.") - args = Attribute( - "A mapping of decoded query argument names as C{str} to " - "corresponding query argument values as C{list}s of C{str}. " - "For example, for a URI with C{'foo=bar&foo=baz&quux=spam'} " - "for its query part, C{args} will be C{{'foo': ['bar', 'baz'], " - "'quux': ['spam']}}.") - - received_headers = Attribute( - "Backwards-compatibility access to C{requestHeaders}. Use " - "C{requestHeaders} instead. C{received_headers} behaves mostly " - "like a C{dict} and does not provide access to all header values.") - - requestHeaders = Attribute( - "A L{http_headers.Headers} instance giving all received HTTP request " - "headers.") - - content = Attribute( - "A file-like object giving the request body. This may be a file on " - "disk, a C{StringIO}, or some other type. The implementation is free " - "to decide on a per-request basis.") - - headers = Attribute( - "Backwards-compatibility access to C{responseHeaders}. Use" - "C{responseHeaders} instead. C{headers} behaves mostly like a " - "C{dict} and does not provide access to all header values nor " - "does it allow multiple values for one header to be set.") - - responseHeaders = Attribute( - "A L{http_headers.Headers} instance holding all HTTP response " - "headers to be sent.") - - def getHeader(key): - """ - Get an HTTP request header. - - @type key: C{str} - @param key: The name of the header to get the value of. - - @rtype: C{str} or C{NoneType} - @return: The value of the specified header, or C{None} if that header - was not present in the request. - """ - - - def getCookie(key): - """ - Get a cookie that was sent from the network. - """ - - - def getAllHeaders(): - """ - Return dictionary mapping the names of all received headers to the last - value received for each. - - Since this method does not return all header information, - C{requestHeaders.getAllRawHeaders()} may be preferred. - """ - - - def getRequestHostname(): - """ - Get the hostname that the user passed in to the request. - - This will either use the Host: header (if it is available) or the - host we are listening on if the header is unavailable. - - @returns: the requested hostname - @rtype: C{str} - """ - - - def getHost(): - """ - Get my originally requesting transport's host. - - @return: An L{IAddress}. - """ - - - def getClientIP(): - """ - Return the IP address of the client who submitted this request. - - @returns: the client IP address or C{None} if the request was submitted - over a transport where IP addresses do not make sense. - @rtype: L{str} or C{NoneType} - """ - - - def getClient(): - """ - Return the hostname of the IP address of the client who submitted this - request, if possible. - - This method is B{deprecated}. See L{getClientIP} instead. - - @rtype: C{NoneType} or L{str} - @return: The canonical hostname of the client, as determined by - performing a name lookup on the IP address of the client. - """ - - - def getUser(): - """ - Return the HTTP user sent with this request, if any. - - If no user was supplied, return the empty string. - - @returns: the HTTP user, if any - @rtype: C{str} - """ - - - def getPassword(): - """ - Return the HTTP password sent with this request, if any. - - If no password was supplied, return the empty string. - - @returns: the HTTP password, if any - @rtype: C{str} - """ - - - def isSecure(): - """ - Return True if this request is using a secure transport. - - Normally this method returns True if this request's HTTPChannel - instance is using a transport that implements ISSLTransport. - - This will also return True if setHost() has been called - with ssl=True. - - @returns: True if this request is secure - @rtype: C{bool} - """ - - - def getSession(sessionInterface=None): - """ - Look up the session associated with this request or create a new one if - there is not one. - - @return: The L{Session} instance identified by the session cookie in - the request, or the C{sessionInterface} component of that session - if C{sessionInterface} is specified. - """ - - - def URLPath(): - """ - @return: A L{URLPath} instance which identifies the URL for which this - request is. - """ - - - def prePathURL(): - """ - @return: At any time during resource traversal, a L{str} giving an - absolute URL to the most nested resource which has yet been - reached. - """ - - - def rememberRootURL(): - """ - Remember the currently-processed part of the URL for later - recalling. - """ - - - def getRootURL(): - """ - Get a previously-remembered URL. - """ - - - # Methods for outgoing response - def finish(): - """ - Indicate that the response to this request is complete. - """ - - - def write(data): - """ - Write some data to the body of the response to this request. Response - headers are written the first time this method is called, after which - new response headers may not be added. - """ - - - def addCookie(k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): - """ - Set an outgoing HTTP cookie. - - In general, you should consider using sessions instead of cookies, see - L{twisted.web.server.Request.getSession} and the - L{twisted.web.server.Session} class for details. - """ - - - def setResponseCode(code, message=None): - """ - Set the HTTP response code. - """ - - - def setHeader(k, v): - """ - Set an HTTP response header. Overrides any previously set values for - this header. - - @type name: C{str} - @param name: The name of the header for which to set the value. - - @type value: C{str} - @param value: The value to set for the named header. - """ - - - def redirect(url): - """ - Utility function that does a redirect. - - The request should have finish() called after this. - """ - - - def setLastModified(when): - """ - Set the C{Last-Modified} time for the response to this request. - - If I am called more than once, I ignore attempts to set Last-Modified - earlier, only replacing the Last-Modified time if it is to a later - value. - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} if appropriate for the time given. - - @param when: The last time the resource being returned was modified, in - seconds since the epoch. - @type when: L{int}, L{long} or L{float} - - @return: If I am a C{If-Modified-Since} conditional request and the time - given is not newer than the condition, I return - L{CACHED} to indicate that you should write no body. - Otherwise, I return a false value. - """ - - - def setETag(etag): - """ - Set an C{entity tag} for the outgoing response. - - That's "entity tag" as in the HTTP/1.1 I{ETag} header, "used for - comparing two or more entities from the same requested resource." - - If I am a conditional request, I may modify my response code to - L{NOT_MODIFIED} or - L{PRECONDITION_FAILED}, if appropriate for the - tag given. - - @param etag: The entity tag for the resource being returned. - @type etag: C{str} - - @return: If I am a C{If-None-Match} conditional request and the tag - matches one in the request, I return L{CACHED} to - indicate that you should write no body. Otherwise, I return a - false value. - """ - - - def setHost(host, port, ssl=0): - """ - Change the host and port the request thinks it's using. - - This method is useful for working with reverse HTTP proxies (e.g. both - Squid and Apache's mod_proxy can do this), when the address the HTTP - client is using is different than the one we're listening on. - - For example, Apache may be listening on https://www.example.com, and - then forwarding requests to http://localhost:8080, but we don't want - HTML produced by Twisted to say 'http://localhost:8080', they should - say 'https://www.example.com', so we do:: - - request.setHost('www.example.com', 443, ssl=1) - """ - - - -class ICredentialFactory(Interface): - """ - A credential factory defines a way to generate a particular kind of - authentication challenge and a way to interpret the responses to these - challenges. It creates - L{ICredentials} providers from - responses. These objects will be used with L{twisted.cred} to authenticate - an authorize requests. - """ - scheme = Attribute( - "A C{str} giving the name of the authentication scheme with which " - "this factory is associated. For example, C{'basic'} or C{'digest'}.") - - - def getChallenge(request): - """ - Generate a new challenge to be sent to a client. - - @type peer: L{twisted.web.http.Request} - @param peer: The request the response to which this challenge will be - included. - - @rtype: C{dict} - @return: A mapping from C{str} challenge fields to associated C{str} - values. - """ - - - def decode(response, request): - """ - Create a credentials object from the given response. - - @type response: C{str} - @param response: scheme specific response string - - @type request: L{twisted.web.http.Request} - @param request: The request being processed (from which the response - was taken). - - @raise twisted.cred.error.LoginFailed: If the response is invalid. - - @rtype: L{twisted.cred.credentials.ICredentials} provider - @return: The credentials represented by the given response. - """ - - - -class IBodyProducer(IPushProducer): - """ - Objects which provide L{IBodyProducer} write bytes to an object which - provides L{IConsumer} by calling its - C{write} method repeatedly. - - L{IBodyProducer} providers may start producing as soon as they have an - L{IConsumer} provider. That is, they - should not wait for a C{resumeProducing} call to begin writing data. - - L{IConsumer.unregisterProducer} - must not be called. Instead, the - L{Deferred} returned from C{startProducing} - must be fired when all bytes have been written. - - L{IConsumer.write} may - synchronously invoke any of C{pauseProducing}, C{resumeProducing}, or - C{stopProducing}. These methods must be implemented with this in mind. - - @since: 9.0 - """ - - # Despite the restrictions above and the additional requirements of - # stopProducing documented below, this interface still needs to be an - # IPushProducer subclass. Providers of it will be passed to IConsumer - # providers which only know about IPushProducer and IPullProducer, not - # about this interface. This interface needs to remain close enough to one - # of those interfaces for consumers to work with it. - - length = Attribute( - """ - C{length} is a C{int} indicating how many bytes in total this - L{IBodyProducer} will write to the consumer or L{UNKNOWN_LENGTH} - if this is not known in advance. - """) - - def startProducing(consumer): - """ - Start producing to the given - L{IConsumer} provider. - - @return: A L{Deferred} which fires with - C{None} when all bytes have been produced or with a - L{Failure} if there is any problem - before all bytes have been produced. - """ - - - def stopProducing(): - """ - In addition to the standard behavior of - L{IProducer.stopProducing} - (stop producing data), make sure the - L{Deferred} returned by - C{startProducing} is never fired. - """ - - - -class IRenderable(Interface): - """ - An L{IRenderable} is an object that may be rendered by the - L{twisted.web.template} templating system. - """ - - def lookupRenderMethod(name): - """ - Look up and return the render method associated with the given name. - - @type name: C{str} - @param name: The value of a render directive encountered in the - document returned by a call to L{IRenderable.render}. - - @return: A two-argument callable which will be invoked with the request - being responded to and the tag object on which the render directive - was encountered. - """ - - - def render(request): - """ - Get the document for this L{IRenderable}. - - @type request: L{IRequest} provider or C{NoneType} - @param request: The request in response to which this method is being - invoked. - - @return: An object which can be flattened. - """ - - - -class ITemplateLoader(Interface): - """ - A loader for templates; something usable as a value for - L{twisted.web.template.Element}'s C{loader} attribute. - """ - - def load(): - """ - Load a template suitable for rendering. - - @return: a C{list} of C{list}s, C{unicode} objects, C{Element}s and - other L{IRenderable} providers. - """ - - - -class IResponse(Interface): - """ - An object representing an HTTP response received from an HTTP server. - - @since: 11.1 - """ - - version = Attribute( - "A three-tuple describing the protocol and protocol version " - "of the response. The first element is of type C{str}, the second " - "and third are of type C{int}. For example, C{('HTTP', 1, 1)}.") - - - code = Attribute("The HTTP status code of this response, as a C{int}.") - - - phrase = Attribute( - "The HTTP reason phrase of this response, as a C{str}.") - - - headers = Attribute("The HTTP response L{Headers} of this response.") - - - length = Attribute( - "The C{int} number of bytes expected to be in the body of this " - "response or L{UNKNOWN_LENGTH} if the server did not indicate how " - "many bytes to expect. For I{HEAD} responses, this will be 0; if " - "the response includes a I{Content-Length} header, it will be " - "available in C{headers}.") - - - def deliverBody(protocol): - """ - Register an L{IProtocol} provider - to receive the response body. - - The protocol will be connected to a transport which provides - L{IPushProducer}. The protocol's C{connectionLost} method will be - called with: - - - ResponseDone, which indicates that all bytes from the response - have been successfully delivered. - - - PotentialDataLoss, which indicates that it cannot be determined - if the entire response body has been delivered. This only occurs - when making requests to HTTP servers which do not set - I{Content-Length} or a I{Transfer-Encoding} in the response. - - - ResponseFailed, which indicates that some bytes from the response - were lost. The C{reasons} attribute of the exception may provide - more specific indications as to why. - """ - - - -class _IRequestEncoder(Interface): - """ - An object encoding data passed to L{IRequest.write}, for example for - compression purpose. - - @since: 12.3 - """ - - def encode(data): - """ - Encode the data given and return the result. - - @param data: The content to encode. - @type data: C{str} - - @return: The encoded data. - @rtype: C{str} - """ - - - def finish(): - """ - Callback called when the request is closing. - - @return: If necessary, the pending data accumulated from previous - C{encode} calls. - @rtype: C{str} - """ - - - -class _IRequestEncoderFactory(Interface): - """ - A factory for returing L{_IRequestEncoder} instances. - - @since: 12.3 - """ - - def encoderForRequest(request): - """ - If applicable, returns a L{_IRequestEncoder} instance which will encode - the request. - """ - - - -UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" - -__all__ = [ - "ICredentialFactory", "IRequest", - "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", - "_IRequestEncoderFactory", - - "UNKNOWN_LENGTH"] diff --git a/setup.py b/setup.py index 92c114a7a..f673b1dc4 100644 --- a/setup.py +++ b/setup.py @@ -41,14 +41,14 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], install_requires=[ - 'Twisted>=10.0.0', + 'Twisted>=13.1.0', 'w3lib>=1.15.0', 'queuelib', 'lxml', 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.3', + 'parsel>=0.9.5', 'PyDispatcher>=2.0.5', 'service_identity', ], diff --git a/tests/pipelines.py b/tests/pipelines.py new file mode 100644 index 000000000..ddfbc7a99 --- /dev/null +++ b/tests/pipelines.py @@ -0,0 +1,11 @@ +""" +Some pipelines used for testing +""" + +class ZeroDivisionErrorPipeline(object): + + def open_spider(self, spider): + a = 1/0 + + def process_item(self, item, spider): + return item diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4843a9a2f..3fa3ed930 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/text')]) self.assertEqual(out.strip(), b'Works') + @defer.inlineCallbacks + def test_redirect_default(self): + _, out, _ = yield self.execute([self.url('/redirect')]) + self.assertEqual(out.strip(), b'Redirected here') + + @defer.inlineCallbacks + def test_redirect_disabled(self): + _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) + err = err.strip() + self.assertIn(b'downloader/response_status_count/302', err, err) + self.assertNotIn(b'downloader/response_status_count/200', err, err) + @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py new file mode 100644 index 000000000..b6d6db9ee --- /dev/null +++ b/tests/test_command_parse.py @@ -0,0 +1,156 @@ +from os.path import join, abspath +from twisted.trial import unittest +from twisted.internet import defer +from scrapy.utils.testsite import SiteTest +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.python import to_native_str +from tests.test_commands import CommandTest + + +class ParseCommandTest(ProcessTest, SiteTest, CommandTest): + command = 'parse' + + def setUp(self): + super(ParseCommandTest, self).setUp() + self.spider_name = 'parse_spider' + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(""" +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class MySpider(scrapy.Spider): + name = '{0}' + + def parse(self, response): + if getattr(self, 'test_arg', None): + self.logger.debug('It Works!') + return [scrapy.Item(), dict(foo='bar')] + + +class MyGoodCrawlSpider(CrawlSpider): + name = 'goodcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + Rule(LinkExtractor(allow=r'/text'), follow=True), + ) + + def parse_item(self, response): + return [scrapy.Item(), dict(foo='bar')] + + def parse(self, response): + return [scrapy.Item(), dict(nomatch='default')] + + +class MyBadCrawlSpider(CrawlSpider): + '''Spider which doesn't define a parse_item callback while using it in a rule.''' + name = 'badcrawl{0}' + + rules = ( + Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), + ) + + def parse(self, response): + return [scrapy.Item(), dict(foo='bar')] +""".format(self.spider_name)) + + fname = abspath(join(self.proj_mod_path, 'pipelines.py')) + with open(fname, 'w') as f: + f.write(""" +import logging + +class MyPipeline(object): + component_name = 'my_pipeline' + + def process_item(self, item, spider): + logging.info('It Works!') + return item +""") + + fname = abspath(join(self.proj_mod_path, 'settings.py')) + with open(fname, 'a') as f: + f.write(""" +ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} +""" % self.project_name) + + @defer.inlineCallbacks + def test_spider_arguments(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-a', 'test_arg=1', + '-c', 'parse', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_pipelines(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--pipelines', + '-c', 'parse', + self.url('/html')]) + self.assertIn("INFO: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_parse_items_no_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_wrong_callback_passed(self): + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find callback""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_callback_set(self): + """If a rule matches the URL, use it's defined callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_crawlspider_matching_rule_default_callback(self): + """If a rule match but it has no callback set, use the 'parse' callback.""" + status, out, stderr = yield self.execute( + ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ) + self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out)) + + @defer.inlineCallbacks + def test_spider_with_no_rules_attribute(self): + """Using -r with a spider with no rule should not produce items.""" + status, out, stderr = yield self.execute( + ['--spider', self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""No CrawlSpider rules found""", to_native_str(stderr)) + + @defer.inlineCallbacks + def test_crawlspider_missing_callback(self): + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + + @defer.inlineCallbacks + def test_crawlspider_no_matching_rule(self): + """The requested URL has no matching rule, so no items should be scraped""" + status, out, stderr = yield self.execute( + ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ) + self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""") + self.assertIn("""Cannot find a rule that matches""", to_native_str(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index c532fc0d8..3e27d6abd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -49,6 +49,35 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) assert out.strip().endswith(b'/redirected') + @defer.inlineCallbacks + def test_redirect_follow_302(self): + _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'200') + + @defer.inlineCallbacks + def test_redirect_not_follow_302(self): + _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'302') + + @defer.inlineCallbacks + def test_fetch_redirect_follow_302(self): + """Test that calling `fetch(url)` follows HTTP redirects by default.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}')" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Redirecting (302)' in errout + assert b'Crawled (200)' in errout + + @defer.inlineCallbacks + def test_fetch_redirect_not_follow_302(self): + """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Crawled (302)' in errout + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') @@ -56,6 +85,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, _ = yield self.execute(['-c', code.format(url)]) self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks + def test_scrapy_import(self): + url = self.url('/text') + code = "fetch(scrapy.Request('{0}'))" + errcode, out, _ = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + @defer.inlineCallbacks def test_local_file(self): filepath = join(tests_datadir, 'test_site/index.html') diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 37e1f2543..2789d207c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -25,5 +25,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute(['-v']) headers = [l.partition(":")[0].strip() for l in out.strip().decode(encoding).splitlines()] - self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'Twisted', - 'Python', 'pyOpenSSL', 'Platform']) + self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', + 'cssselect', 'parsel', 'w3lib', + 'Twisted', 'Python', 'pyOpenSSL', + 'Platform']) diff --git a/tests/test_commands.py b/tests/test_commands.py index d25045cb9..922098668 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -38,11 +38,11 @@ class ProjectTest(unittest.TestCase): return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs) - def proc(self, *new_args, **kwargs): + def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args p = subprocess.Popen(args, cwd=self.cwd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs) + **popen_kwargs) waited = 0 interval = 0.2 @@ -182,6 +182,17 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug("It Works!") + return [] +""" + @contextmanager def _create_file(self, content, name): tmpdir = self.mktemp() @@ -194,32 +205,44 @@ class RunSpiderCommandTest(CommandTest): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py'): + def runspider(self, code, name='myspider.py', args=()): with self._create_file(code, name) as fname: - return self.proc('runspider', fname) + return self.proc('runspider', fname, *args) + + def get_log(self, code, name='myspider.py', args=()): + p = self.runspider(code, name=name, args=args) + return to_native_str(p.stderr.read()) def test_runspider(self): - spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - def start_requests(self): - self.logger.debug("It Works!") - return [] -""" - p = self.runspider(spider) - log = to_native_str(p.stderr.read()) - + log = self.get_log(self.debug_log_spider) self.assertIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_LEVEL=INFO')) + self.assertNotIn("DEBUG: It Works!", log) + self.assertIn("INFO: Spider opened", log) + + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=1')) + print(log1) + self.assertIn("[myspider] DEBUG: It Works!", log1) + self.assertIn("[scrapy]", log1) + self.assertNotIn("[scrapy.core.engine]", log1) + + log2 = self.get_log(self.debug_log_spider, + args=('-s', 'LOG_SHORT_NAMES=0')) + print(log2) + self.assertIn("[myspider] DEBUG: It Works!", log2) + self.assertNotIn("[scrapy]", log2) + self.assertIn("[scrapy.core.engine]", log2) + def test_runspider_no_spider_found(self): - p = self.runspider("from scrapy.spiders import Spider\n") - log = to_native_str(p.stderr.read()) + log = self.get_log("from scrapy.spiders import Spider\n") self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): @@ -228,12 +251,11 @@ class MySpider(scrapy.Spider): self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - p = self.runspider('', 'myspider.txt') - log = to_native_str(p.stderr.read()) + log = self.get_log('', name='myspider.txt') self.assertIn('Unable to load', log) def test_start_requests_errors(self): - p = self.runspider(""" + log = self.get_log(""" import scrapy class BadSpider(scrapy.Spider): @@ -241,76 +263,11 @@ class BadSpider(scrapy.Spider): def start_requests(self): raise Exception("oops!") """, name="badspider.py") - log = to_native_str(p.stderr.read()) print(log) self.assertIn("start_requests", log) self.assertIn("badspider.py", log) -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' - - def setUp(self): - super(ParseCommandTest, self).setUp() - self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(""" -import scrapy - -class MySpider(scrapy.Spider): - name = '{0}' - - def parse(self, response): - if getattr(self, 'test_arg', None): - self.logger.debug('It Works!') - return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) - - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" -import logging - -class MyPipeline(object): - component_name = 'my_pipeline' - - def process_item(self, item, spider): - logging.info('It Works!') - return item -""") - - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) - - @defer.inlineCallbacks - def test_spider_arguments(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-a', 'test_arg=1', - '-c', 'parse', - self.url('/html')]) - self.assertIn("DEBUG: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_pipelines(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--pipelines', - '-c', 'parse', - self.url('/html')]) - self.assertIn("INFO: It Works!", to_native_str(stderr)) - - @defer.inlineCallbacks - def test_parse_items(self): - status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] - ) - self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out)) - - - class BenchCommandTest(CommandTest): def test_run(self): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 90fd921c8..1b4a4b3b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -250,6 +250,19 @@ with multiples lines yield self.assertFailure(crawler.crawl(), TestError) self.assertFalse(crawler.crawling) + @defer.inlineCallbacks + def test_open_spider_error_on_faulty_pipeline(self): + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) + yield self.assertFailure( + self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + ZeroDivisionError) + self.assertFalse(crawler.crawling) + @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 9db073cc5..e8c92affa 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase): latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' self.assertEquals(perc_encoded_utf8_url, req_result.url) - def test_location_with_wrong_encoding(self): + def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação' # header with wrong encoding (utf-8) + utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' self.assertEquals(perc_encoded_utf8_url, req_result.url) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 3de9399cf..eb17974bf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -3,10 +3,10 @@ from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError +from twisted.web.client import ResponseFailed from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.xlib.tx import ResponseFailed from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f2e94e171..60306eacb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import import re from twisted.internet import reactor, error @@ -30,11 +31,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', b''' + ROBOTS = re.sub(b'^\s+(?m)', b'', u''' User-Agent: * Disallow: /admin/ Disallow: /static/ - ''') + + # taken from https://en.wikipedia.org/robots.txt + Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: + Disallow: /wiki/Käyttäjä: + + User-Agent: UnicödeBöt + Disallow: /some/randome/page.html + '''.encode('utf-8')) response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): deferred = Deferred() @@ -48,7 +56,9 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): return DeferredList([ self.assertNotIgnored(Request('http://site.local/allowed'), middleware), self.assertIgnored(Request('http://site.local/admin/main'), middleware), - self.assertIgnored(Request('http://site.local/static/'), middleware) + self.assertIgnored(Request('http://site.local/static/'), middleware), + self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), + self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 353b21927..e93d2bafb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + class TestSpider(scrapy.Spider): + name = 'testspider' + start_urls = ['http://localhost:8998/'] + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, ordered=False) + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + formats = ('json', + 'jsonlines', + 'xml', + 'csv',) + + for fmt in formats: + settings = {'FEED_FORMAT': fmt} + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[\n\n]'), + ('jsonlines', b''), + ('xml', b'\n'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + data = yield self.exported_no_data(settings) + self.assertEqual(data, expctd) + @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase): def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] header = ['foo'] - + formats = { 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - + for format in formats: settings = {'FEED_FORMAT': format} data = yield self.exported_data(items, settings) self.assertEqual(formats[format], data) - + formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - + for format in formats: settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} data = yield self.exported_data(items, settings) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index c7f36687a..7624aa4c4 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -7,6 +7,7 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector from scrapy.utils.python import to_native_str +from scrapy.exceptions import NotSupported class BaseResponseTest(unittest.TestCase): @@ -127,6 +128,18 @@ class BaseResponseTest(unittest.TestCase): absolute = 'http://www.example.com/test' self.assertEqual(joined, absolute) + def test_shortcut_attributes(self): + r = self.response_class("http://example.com", body=b'hello') + if self.response_class == Response: + msg = "Response content isn't text" + self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text') + self.assertRaisesRegexp(NotSupported, msg, r.css, 'body') + self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body') + else: + r.text + r.css('body') + r.xpath('//body') + class TextResponseTest(BaseResponseTest): diff --git a/tests/test_mail.py b/tests/test_mail.py index bd7e49621..b139e98d8 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -10,7 +10,8 @@ class MailSenderTest(unittest.TestCase): def test_send(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', _callback=self._catch_mail_sent) + mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', + _callback=self._catch_mail_sent) assert self.catched_msg @@ -24,9 +25,16 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get_payload(), 'body') self.assertEqual(msg.get('Content-Type'), 'text/plain') + def test_send_single_values_to_and_cc(self): + mailsender = MailSender(debug=True) + mailsender.send(to='test@scrapy.org', subject='subject', body='body', + cc='test@scrapy.org', _callback=self._catch_mail_sent) + def test_send_html(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='

      body

      ', mimetype='text/html', _callback=self._catch_mail_sent) + mailsender.send(to=['test@scrapy.org'], subject='subject', + body='

      body

      ', mimetype='text/html', + _callback=self._catch_mail_sent) msg = self.catched_msg['msg'] self.assertEqual(msg.get_payload(), '

      body

      ') @@ -90,7 +98,8 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, - attachs=attachs, charset='utf-8', _callback=self._catch_mail_sent) + attachs=attachs, charset='utf-8', + _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['subject'], subject) @@ -99,7 +108,8 @@ class MailSenderTest(unittest.TestCase): msg = self.catched_msg['msg'] self.assertEqual(msg['subject'], subject) self.assertEqual(msg.get_charset(), Charset('utf-8')) - self.assertEqual(msg.get('Content-Type'), 'multipart/mixed; charset="utf-8"') + self.assertEqual(msg.get('Content-Type'), + 'multipart/mixed; charset="utf-8"') payload = msg.get_payload() assert isinstance(payload, list) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bda2a2199..e3ec04b8d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -208,7 +208,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "FILES_EXPIRES": random.randint(1, 1000), + "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), "FILES_STORE": self.tempdir @@ -255,16 +255,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If file settings are defined but they are not defined for subclass class attributes - should be preserved. + If file settings are defined but they are not defined for subclass + settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) + setting_value = settings.get(settings_attr) self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + self.assertEqual(value, setting_value) def test_no_custom_settings_for_subclasses(self): """ @@ -321,6 +322,24 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_urls_field, "that") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of FilesPipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(FilesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), + expected_value) + + class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8286582de..342f25ea9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -244,7 +244,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) settings = { - "IMAGES_EXPIRES": random.randint(1, 1000), + "IMAGES_EXPIRES": random.randint(100, 1000), "IMAGES_STORE": self.tempdir, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), @@ -309,17 +309,19 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): def test_subclass_attrs_preserved_custom_settings(self): """ - If image settings are defined but they are not defined for subclass class attributes - should be preserved. + If image settings are defined but they are not defined for subclass default + values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() settings = self._generate_fake_settings() pipeline = pipeline_cls.from_settings(Settings(settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: - # Instance attribute (lowercase) must be equal to class attribute (uppercase). + # Instance attribute (lowercase) must be equal to + # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(value, getattr(pipeline, pipe_attr)) + setings_value = settings.get(settings_attr) + self.assertEqual(value, setings_value) def test_no_custom_settings_for_subclasses(self): """ @@ -370,11 +372,26 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) self.assertEqual(pipeline.images_result_field, "something_else") self.assertEqual(pipeline.images_urls_field, "something") + def test_user_defined_subclass_default_key_names(self): + """Test situation when user defines subclass of ImagePipeline, + but uses attribute names for default pipeline (without prefixing + them with pipeline class name). + """ + settings = self._generate_fake_settings() + + class UserPipe(ImagesPipeline): + pass + + pipeline_cls = UserPipe.from_settings(Settings(settings)) + + for pipe_attr, settings_attr in self.img_cls_attribute_names: + expected_value = settings.get(settings_attr) + self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), + expected_value) def _create_image(format, *a, **kw): buf = TemporaryFile() diff --git a/tests/test_spider.py b/tests/test_spider.py index 1d22c1212..079734a69 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -332,13 +332,17 @@ class SitemapSpiderTest(SpiderTest): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://example.com/sitemap.xml', - 'http://example.com/sitemap-product-index.xml']) + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://www.example.com/sitemap-relative-url.xml']) class BaseSpiderDeprecationTest(unittest.TestCase): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 68dca2e98..b2ad93b3f 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,6 +1,7 @@ import sys import os import shutil +import warnings from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -9,6 +10,7 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy +import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings @@ -22,8 +24,7 @@ class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = self.mktemp() - os.mkdir(self.tmpdir) + self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) @@ -40,7 +41,7 @@ class SpiderLoaderTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spider_loader.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): spider1 = self.spider_loader.load("spider1") @@ -66,7 +67,7 @@ class SpiderLoaderTest(unittest.TestCase): self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 1 - def test_load_spider_module(self): + def test_load_spider_module_multiple(self): prefix = 'tests.test_spiderloader.test_spiders.' module = ','.join(prefix + s for s in ('spider1', 'spider2')) settings = Settings({'SPIDER_MODULES': module}) @@ -89,3 +90,14 @@ class SpiderLoaderTest(unittest.TestCase): crawler = runner.create_crawler('spider1') self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + + def test_bad_spider_modules_warning(self): + + with warnings.catch_warnings(record=True) as w: + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + spider_loader = SpiderLoader.from_settings(settings) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) diff --git a/tests/test_spiderloader/test_spiders/nested/__init__.py b/tests/test_spiderloader/test_spiders/nested/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py new file mode 100644 index 000000000..35b71870a --- /dev/null +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -0,0 +1,9 @@ +from scrapy.spiders import Spider + +class Spider4(Spider): + name = "spider4" + allowed_domains = ['spider4.com'] + + @classmethod + def handles_request(cls, request): + return request.url == 'http://spider4.com/onlythis' diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index a64400482..319746350 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,3 +1,4 @@ +import logging from unittest import TestCase from testfixtures import LogCapture @@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): self.assertIn('Ignoring response <500', str(log)) self.assertNotIn('Ignoring response <200', str(log)) self.assertNotIn('Ignoring response <402', str(log)) + + @defer.inlineCallbacks + def test_logging_level(self): + # HttpError logs ignored responses with level INFO + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.INFO) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertIn('Ignoring response <402', str(log)) + self.assertIn('Ignoring response <404', str(log)) + self.assertIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log)) + + # with level WARNING, we shouldn't capture anything from HttpError + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.WARNING) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertNotIn('Ignoring response <402', str(log)) + self.assertNotIn('Ignoring response <404', str(log)) + self.assertNotIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log)) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b31d2179c..80f797227 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy import unittest -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaselessDict, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] @@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase): assert isinstance(h2, CaselessDict) +class SequenceExcludeTest(unittest.TestCase): + + def test_list(self): + seq = [1, 2, 3] + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn(4, d) + self.assertNotIn(2, d) + + def test_range(self): + seq = range(10, 20) + d = SequenceExclude(seq) + self.assertIn(5, d) + self.assertIn(20, d) + self.assertNotIn(15, d) + + def test_six_range(self): + import six.moves + seq = six.moves.range(10**3, 10**6) + d = SequenceExclude(seq) + self.assertIn(10**2, d) + self.assertIn(10**7, d) + self.assertNotIn(10**4, d) + + def test_range_step(self): + seq = range(10, 20, 3) + d = SequenceExclude(seq) + are_not_in = [v for v in range(10, 20, 3) if v in d] + self.assertEquals([], are_not_in) + + are_not_in = [v for v in range(10, 20) if v in d] + self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + + def test_string_seq(self): + seq = "cde" + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_stringset_seq(self): + seq = set("cde") + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_set(self): + """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" + seq = set([-3, "test", 1.1]) + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn("foo", d) + self.assertIn(3.14, d) + self.assertIn(set("bar"), d) + + # supplied sequence is a set, so checking for list (non)inclusion fails + self.assertRaises(TypeError, (0, 1, 2) in d) + self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + + for v in [-3, "test", 1.1]: + self.assertNotIn(v, d) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py new file mode 100644 index 000000000..1239195e8 --- /dev/null +++ b/tests/test_utils_project.py @@ -0,0 +1,35 @@ +import unittest +import os +import tempfile +import shutil +import contextlib +from scrapy.utils.project import data_path + + +@contextlib.contextmanager +def inside_a_project(): + prev_dir = os.getcwd() + project_dir = tempfile.mkdtemp() + + try: + os.chdir(project_dir) + with open('scrapy.cfg', 'w') as f: + # create an empty scrapy.cfg + f.close() + + yield project_dir + finally: + os.chdir(prev_dir) + shutil.rmtree(project_dir) + + +class ProjectUtilsTest(unittest.TestCase): + def test_data_path_outside_project(self): + self.assertEquals('.scrapy/somepath', data_path('somepath')) + self.assertEquals('/absolute/path', data_path('/absolute/path')) + + def test_data_path_inside_project(self): + with inside_a_project() as proj_path: + expected = os.path.join(proj_path, '.scrapy', 'somepath') + self.assertEquals(expected, data_path('somepath')) + self.assertEquals('/absolute/path', data_path('/absolute/path')) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index bd2677956..716bb44eb 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -119,13 +119,18 @@ Disallow: /s*/*tags # Sitemap files Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml # Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), - ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')), + ['http://example.com/sitemap.xml', + 'http://example.com/sitemap-product-index.xml', + 'http://example.com/sitemap-uppercase.xml', + 'http://example.com/sitemap-relative-url.xml']) def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before tag""" diff --git a/tox.ini b/tox.ini index f6de64b27..812302b4c 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,20 @@ deps = zope.interface==3.6.1 -rtests/requirements.txt +[testenv:jessie] +# https://packages.debian.org/en/jessie/python/ +# https://packages.debian.org/en/jessie/zope/ +basepython = python2.7 +deps = + pyOpenSSL==0.14 + lxml==3.4.0 + Twisted==14.0.2 + boto==2.34.0 + Pillow==2.6.1 + cssselect==0.9.1 + zope.interface==4.1.1 + -rtests/requirements.txt + [testenv:trunk] basepython = python2.7 commands =