mirror of https://github.com/scrapy/scrapy.git
Merge branch 'release-1.0.4' into 1.0
This commit is contained in:
Daniel Graña
commit
0797680bc4
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
set -x
|
||||
|
||||
if [[ "${TOXENV}" == "pypy" ]]; then
|
||||
sudo add-apt-repository -y ppa:pypy/ppa
|
||||
sudo apt-get -qy update
|
||||
sudo apt-get install -y pypy pypy-dev
|
||||
# This is required because we need to get rid of the Travis installed PyPy
|
||||
# or it'll take precedence over the PPA installed one.
|
||||
sudo rm -rf /usr/local/pypy/bin
|
||||
fi
|
||||
|
||||
# Workaround travis-ci/travis-ci#2065
|
||||
pip install -U wheel
|
||||
|
|
@ -1,12 +1,16 @@
|
|||
language: python
|
||||
python: 2.7
|
||||
sudo: false
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d\.\d+$/
|
||||
env:
|
||||
- TOXENV=py27
|
||||
- TOXENV=precise
|
||||
- TOXENV=py33
|
||||
- TOXENV=docs
|
||||
install:
|
||||
- "./.travis-workarounds.sh"
|
||||
- pip install -U tox twine wheel
|
||||
script: tox
|
||||
notifications:
|
||||
|
|
@ -15,6 +19,9 @@ notifications:
|
|||
skip_join: true
|
||||
channels:
|
||||
- irc.freenode.org#scrapy
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/pip
|
||||
deploy:
|
||||
provider: pypi
|
||||
distributions: "sdist bdist_wheel"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ Scrapy
|
|||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: PyPI Version
|
||||
|
||||
.. image:: https://img.shields.io/pypi/dm/Scrapy.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: PyPI Monthly downloads
|
||||
|
||||
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
|
||||
:target: http://travis-ci.org/scrapy/scrapy
|
||||
:alt: Build Status
|
||||
|
|
@ -13,6 +17,11 @@ Scrapy
|
|||
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: Wheel Status
|
||||
|
||||
.. image:: http://static.scrapy.org/py3progress/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
|
||||
:alt: Python 3 Porting Status
|
||||
|
||||
|
||||
Overview
|
||||
========
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import glob
|
||||
import six
|
||||
import pytest
|
||||
from twisted import version as twisted_version
|
||||
|
||||
|
||||
def _py_files(folder):
|
||||
|
|
@ -21,6 +22,9 @@ collect_ignore = [
|
|||
"scrapy/spider.py",
|
||||
] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
|
||||
|
||||
if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
|
||||
collect_ignore += _py_files("scrapy/xlib/tx")
|
||||
|
||||
|
||||
if six.PY3:
|
||||
for line in open('tests/py3-ignores.txt'):
|
||||
|
|
|
|||
|
|
@ -108,7 +108,10 @@ html_theme = 'sphinx_rtd_theme'
|
|||
#html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = []
|
||||
# Add path to the RTD explicitly to robustify builds (otherwise might
|
||||
# fail in a clean Debian build env)
|
||||
import sphinx_rtd_theme
|
||||
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
|
||||
# The style sheet to use for HTML and HTML Help pages. A file of that name
|
||||
|
|
|
|||
|
|
@ -146,6 +146,14 @@ tests requires `tox`_.
|
|||
Running tests
|
||||
-------------
|
||||
|
||||
Make sure you have a recent enough `tox`_ installation:
|
||||
|
||||
``tox --version``
|
||||
|
||||
If your version is older than 1.7.0, please update it first:
|
||||
|
||||
``pip install -U tox``
|
||||
|
||||
To run all tests go to the root directory of Scrapy source code and run:
|
||||
|
||||
``tox``
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ I get "Filtered offsite request" messages. How can I fix them?
|
|||
Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a
|
||||
problem, so you may not need to fix them.
|
||||
|
||||
Those message are thrown by the Offsite Spider Middleware, which is a spider
|
||||
Those messages are thrown by the Offsite Spider Middleware, which is a spider
|
||||
middleware (enabled by default) whose purpose is to filter out requests to
|
||||
domains outside the ones covered by the spider.
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ First steps
|
|||
===========
|
||||
|
||||
.. toctree::
|
||||
:caption: First steps
|
||||
:hidden:
|
||||
|
||||
intro/overview
|
||||
|
|
@ -53,6 +54,7 @@ Basic concepts
|
|||
==============
|
||||
|
||||
.. toctree::
|
||||
:caption: Basic concepts
|
||||
:hidden:
|
||||
|
||||
topics/commands
|
||||
|
|
@ -110,6 +112,7 @@ Built-in services
|
|||
=================
|
||||
|
||||
.. toctree::
|
||||
:caption: Built-in services
|
||||
:hidden:
|
||||
|
||||
topics/logging
|
||||
|
|
@ -138,6 +141,7 @@ Solving specific problems
|
|||
=========================
|
||||
|
||||
.. toctree::
|
||||
:caption: Solving specific problems
|
||||
:hidden:
|
||||
|
||||
faq
|
||||
|
|
@ -203,6 +207,7 @@ Extending Scrapy
|
|||
================
|
||||
|
||||
.. toctree::
|
||||
:caption: Extending Scrapy
|
||||
:hidden:
|
||||
|
||||
topics/architecture
|
||||
|
|
@ -240,6 +245,7 @@ All the rest
|
|||
============
|
||||
|
||||
.. toctree::
|
||||
:caption: All the rest
|
||||
:hidden:
|
||||
|
||||
news
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ The installation steps assume that you have the following things installed:
|
|||
* `Python`_ 2.7
|
||||
|
||||
* `pip`_ and `setuptools`_ Python packages. Nowadays `pip`_ requires and
|
||||
installs `setuptools`_ if not installed.
|
||||
installs `setuptools`_ if not installed. Python 2.7.9 and later include
|
||||
`pip`_ by default, so you may have it already.
|
||||
|
||||
* `lxml`_. Most Linux distributions ships prepackaged versions of lxml.
|
||||
Otherwise refer to http://lxml.de/installation.html
|
||||
|
|
@ -23,9 +24,7 @@ The installation steps assume that you have the following things installed:
|
|||
where the Python installer ships it bundled.
|
||||
|
||||
You can install Scrapy using pip (which is the canonical way to install Python
|
||||
packages).
|
||||
|
||||
To install using pip::
|
||||
packages). To install using ``pip`` run::
|
||||
|
||||
pip install Scrapy
|
||||
|
||||
|
|
@ -34,6 +33,22 @@ To install using pip::
|
|||
Platform specific installation notes
|
||||
====================================
|
||||
|
||||
Anaconda
|
||||
--------
|
||||
|
||||
.. note::
|
||||
|
||||
For Windows users, or if you have issues installing through `pip`, this is
|
||||
the recommended way to install Scrapy.
|
||||
|
||||
If you already have installed `Anaconda`_ or `Miniconda`_, the company
|
||||
`Scrapinghub`_ maintains official conda packages for Linux, Windows and OS X.
|
||||
|
||||
To install Scrapy using ``conda``, run::
|
||||
|
||||
conda install -c scrapinghub scrapy
|
||||
|
||||
|
||||
Windows
|
||||
-------
|
||||
|
||||
|
|
@ -58,7 +73,8 @@ Windows
|
|||
|
||||
Be sure you download the architecture (win32 or amd64) that matches your system
|
||||
|
||||
* Install `pip`_ from https://pip.pypa.io/en/latest/installing.html
|
||||
* *(Only required for Python<2.7.9)* Install `pip`_ from
|
||||
https://pip.pypa.io/en/latest/installing.html
|
||||
|
||||
Now open a Command prompt to check ``pip`` is installed correctly::
|
||||
|
||||
|
|
@ -79,6 +95,21 @@ Instead, use the official :ref:`Ubuntu Packages <topics-ubuntu>`, which already
|
|||
solve all dependencies for you and are continuously updated with the latest bug
|
||||
fixes.
|
||||
|
||||
If you prefer to build the python dependencies locally instead of relying on
|
||||
system packages you'll need to install their required non-python dependencies
|
||||
first::
|
||||
|
||||
sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
|
||||
You can install Scrapy with ``pip`` after that::
|
||||
|
||||
pip install Scrapy
|
||||
|
||||
.. note::
|
||||
|
||||
The same non-python dependencies can be used to install Scrapy in Debian
|
||||
Wheezy (7.0) and above.
|
||||
|
||||
Archlinux
|
||||
---------
|
||||
|
||||
|
|
@ -86,6 +117,58 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa
|
|||
|
||||
yaourt -S scrapy
|
||||
|
||||
Mac OS X
|
||||
--------
|
||||
|
||||
Building Scrapy's dependencies requires the presence of a C compiler and
|
||||
development headers. On OS X this is typically provided by Apple’s Xcode
|
||||
development tools. To install the Xcode command line tools open a terminal
|
||||
window and run::
|
||||
|
||||
xcode-select --install
|
||||
|
||||
There's a `known issue <https://github.com/pypa/pip/issues/2468>`_ that
|
||||
prevents ``pip`` from updating system packages. This has to be addressed to
|
||||
successfully install Scrapy and its dependencies. Here are some proposed
|
||||
solutions:
|
||||
|
||||
* *(Recommended)* **Don't** use system python, install a new, updated version
|
||||
that doesn't conflict with the rest of your system. Here's how to do it using
|
||||
the `homebrew`_ package manager:
|
||||
|
||||
* Install `homebrew`_ following the instructions in http://brew.sh/
|
||||
|
||||
* Update your ``PATH`` variable to state that homebrew packages should be
|
||||
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
|
||||
if you're using `zsh`_ as default shell)::
|
||||
|
||||
echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
|
||||
|
||||
* Reload ``.bashrc`` to ensure the changes have taken place::
|
||||
|
||||
source ~/.bashrc
|
||||
|
||||
* Install python::
|
||||
|
||||
brew install python
|
||||
|
||||
* Latest versions of python have ``pip`` bundled with them so you won't need
|
||||
to install it separately. If this is not the case, upgrade python::
|
||||
|
||||
brew update; brew upgrade python
|
||||
|
||||
* *(Optional)* Install Scrapy inside an isolated python environment.
|
||||
|
||||
This method is a workaround for the above OS X issue, but it's an overall
|
||||
good practice for managing dependencies and can complement the first method.
|
||||
|
||||
`virtualenv`_ is a tool you can use to create virtual environments in python.
|
||||
We recommended reading a tutorial like
|
||||
http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
|
||||
|
||||
After any of these workarounds you should be able to install Scrapy::
|
||||
|
||||
pip install Scrapy
|
||||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _pip: https://pip.pypa.io/en/latest/installing.html
|
||||
|
|
@ -95,3 +178,9 @@ You can follow the generic instructions or install Scrapy from `AUR Scrapy packa
|
|||
.. _OpenSSL: https://pypi.python.org/pypi/pyOpenSSL
|
||||
.. _setuptools: https://pypi.python.org/pypi/setuptools
|
||||
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
|
||||
.. _homebrew: http://brew.sh/
|
||||
.. _zsh: http://www.zsh.org/
|
||||
.. _virtualenv: https://virtualenv.pypa.io/en/latest/
|
||||
.. _Scrapinghub: http://scrapinghub.com
|
||||
.. _Anaconda: http://docs.continuum.io/anaconda/index
|
||||
.. _Miniconda: http://conda.pydata.org/docs/install/quick.html
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ These are some common properties often found in broad crawls:
|
|||
|
||||
As said above, Scrapy default settings are optimized for focused crawls, not
|
||||
broad crawls. However, due to its asynchronous architecture, Scrapy is very
|
||||
well suited for performing fast broad crawls. This page summarize some things
|
||||
well suited for performing fast broad crawls. This page summarizes some things
|
||||
you need to keep in mind when using Scrapy for doing broad crawls, along with
|
||||
concrete suggestions of Scrapy settings to tune in order to achieve an
|
||||
efficient broad crawl.
|
||||
|
|
@ -46,7 +46,7 @@ Concurrency is the number of requests that are processed in parallel. There is
|
|||
a global limit and a per-domain limit.
|
||||
|
||||
The default global concurrency limit in Scrapy is not suitable for crawling
|
||||
many different domains in parallel, so you will want to increase it. How much
|
||||
many different domains in parallel, so you will want to increase it. How much
|
||||
to increase it will depend on how much CPU you crawler will have available. A
|
||||
good starting point is ``100``, but the best way to find out is by doing some
|
||||
trials and identifying at what concurrency your Scrapy process gets CPU
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ This section describes the different options you have for deploying your Scrapy
|
|||
spiders to run them on a regular basis. Running Scrapy spiders in your local
|
||||
machine is very convenient for the (early) development stage, but not so much
|
||||
when you need to execute long-running spiders or move spiders to run in
|
||||
production continously. This is where the solutions for deploying Scrapy
|
||||
production continuously. This is where the solutions for deploying Scrapy
|
||||
spiders come in.
|
||||
|
||||
Popular choices for deploying Scrapy spiders are:
|
||||
|
|
|
|||
|
|
@ -736,7 +736,7 @@ RetryMiddleware
|
|||
|
||||
.. class:: RetryMiddleware
|
||||
|
||||
A middlware to retry failed requests that are potentially caused by
|
||||
A middleware to retry failed requests that are potentially caused by
|
||||
temporary problems such as a connection timeout or HTTP 500 error.
|
||||
|
||||
Failed pages are collected on the scraping process and rescheduled at the
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ remain disabled. Those components include:
|
|||
|
||||
* Extensions
|
||||
* Item pipelines
|
||||
* Downloader middlwares
|
||||
* Downloader middlewares
|
||||
* Spider middlewares
|
||||
|
||||
The exception must be raised in the component constructor.
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ Extensions use the :ref:`Scrapy settings <topics-settings>` to manage their
|
|||
settings, just like any other Scrapy code.
|
||||
|
||||
It is customary for extensions to prefix their settings with their own name, to
|
||||
avoid collision with existing (and future) extensions. For example, an
|
||||
avoid collision with existing (and future) extensions. For example, a
|
||||
hypothetic extension to handle `Google Sitemaps`_ would use settings like
|
||||
`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ Here is the code of such extension::
|
|||
self.items_scraped += 1
|
||||
if self.items_scraped % self.item_count == 0:
|
||||
logger.info("scraped %d items", self.items_scraped)
|
||||
|
||||
|
||||
|
||||
.. _topics-extensions-ref:
|
||||
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ they work as we expect.
|
|||
|
||||
As you can see, the page markup is not very descriptive: the elements don't
|
||||
contain ``id``, ``class`` or any attribute that clearly identifies them, so
|
||||
we''ll use the ranking bars as a reference point to select the data to extract
|
||||
we'll use the ranking bars as a reference point to select the data to extract
|
||||
when we construct our XPaths.
|
||||
|
||||
After using FireBug, we can see that each link is inside a ``td`` tag, which is
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ contain a price::
|
|||
Write items to a JSON file
|
||||
--------------------------
|
||||
|
||||
The following pipeline stores all scraped items (from all spiders) into a a
|
||||
The following pipeline stores all scraped items (from all spiders) into a
|
||||
single ``items.jl`` file, containing one item per line serialized in JSON
|
||||
format::
|
||||
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ the example above.
|
|||
You can specify any kind of metadata for each field. There is no restriction on
|
||||
the values accepted by :class:`Field` objects. For this same
|
||||
reason, there is no reference list of all available metadata keys. Each key
|
||||
defined in :class:`Field` objects could be used by a different components, and
|
||||
defined in :class:`Field` objects could be used by a different component, and
|
||||
only those components know about it. You can also define and use any other
|
||||
:class:`Field` key in your project too, for your own needs. The main goal of
|
||||
:class:`Field` objects is to provide a way to define all field metadata in one
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ subclasses):
|
|||
A real example
|
||||
--------------
|
||||
|
||||
Let's see a concrete example of an hypothetical case of memory leaks.
|
||||
Let's see a concrete example of a hypothetical case of memory leaks.
|
||||
Suppose we have some spider with a line similar to this one::
|
||||
|
||||
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Downloading and processing files and images
|
|||
.. currentmodule:: scrapy.pipelines.images
|
||||
|
||||
Scrapy provides reusable :doc:`item pipelines </topics/item-pipeline>` for
|
||||
downloading fies attached to a particular item (for example, when you scrape
|
||||
downloading files attached to a particular item (for example, when you scrape
|
||||
products and also want to download their images locally). These pipelines share
|
||||
a bit of functionality and structure (we refer to them as media pipelines), but
|
||||
typically you'll either use the Files Pipeline or the Images Pipeline.
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ project as example.
|
|||
process = CrawlerProcess(get_project_settings())
|
||||
|
||||
# 'followall' is the name of one of the spiders of the project.
|
||||
process.crawl('testspider', domain='scrapinghub.com')
|
||||
process.crawl('followall', domain='scrapinghub.com')
|
||||
process.start() # the script will block here until the crawling is finished
|
||||
|
||||
There's another Scrapy utility that provides more control over the crawling
|
||||
|
|
@ -228,7 +228,7 @@ with varying degrees of sophistication. Getting around those measures can be
|
|||
difficult and tricky, and may sometimes require special infrastructure. Please
|
||||
consider contacting `commercial support`_ if in doubt.
|
||||
|
||||
Here are some tips to keep in mind when dealing with these kind of sites:
|
||||
Here are some tips to keep in mind when dealing with these kinds of sites:
|
||||
|
||||
* rotate your user agent from a pool of well-known ones from browsers (google
|
||||
around to get a list of them)
|
||||
|
|
|
|||
|
|
@ -146,12 +146,12 @@ If you want to extract only first matched element, you can call the selector ``.
|
|||
|
||||
It returns ``None`` if no element was found:
|
||||
|
||||
>>> response.xpath('//div/[id="not-exists"]/text()').extract_first() is None
|
||||
>>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
|
||||
True
|
||||
|
||||
A default return value can be provided as an argument, to be used instead of ``None``:
|
||||
|
||||
>>> sel.xpath('//div/[id="not-exists"]/text()').extract_first(default='not-found')
|
||||
>>> sel.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
|
||||
'not-found'
|
||||
|
||||
Notice that CSS selectors can select text or attribute nodes using CSS3
|
||||
|
|
@ -579,7 +579,7 @@ Built-in Selectors reference
|
|||
is used together with ``text``.
|
||||
|
||||
If ``type`` is ``None`` and a ``response`` is passed, the selector type is
|
||||
inferred from the response type as follow:
|
||||
inferred from the response type as follows:
|
||||
|
||||
* ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
|
||||
* ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
|
||||
|
|
@ -757,7 +757,7 @@ nodes can be accessed directly by their names::
|
|||
<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
|
||||
...
|
||||
|
||||
If you wonder why the namespace removal procedure isn't called always by default
|
||||
If you wonder why the namespace removal procedure isn't always called by default
|
||||
instead of having to call it manually, this is because of two reasons, which, in order
|
||||
of relevance, are:
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ Available Shortcuts
|
|||
* ``view(response)`` - open the given response in your local web browser, for
|
||||
inspection. This will add a `\<base\> tag`_ to the response body in order
|
||||
for external links (such as images and style sheets) to display properly.
|
||||
Note, however,that this will create a temporary file in your computer,
|
||||
Note, however, that this will create a temporary file in your computer,
|
||||
which won't be removed automatically.
|
||||
|
||||
.. _<base> tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ scrapy.Spider
|
|||
|
||||
.. attribute:: settings
|
||||
|
||||
Configuration on which this spider is been ran. This is a
|
||||
Configuration for running this spider. This is a
|
||||
:class:`~scrapy.settings.Settings` instance, see the
|
||||
:ref:`topics-settings` topic for a detailed introduction on this subject.
|
||||
|
||||
|
|
@ -122,12 +122,12 @@ scrapy.Spider
|
|||
|
||||
This is the class method used by Scrapy to create your spiders.
|
||||
|
||||
You probably won't need to override this directly, since the default
|
||||
You probably won't need to override this directly because the default
|
||||
implementation acts as a proxy to the :meth:`__init__` method, calling
|
||||
it with the given arguments `args` and named arguments `kwargs`.
|
||||
|
||||
Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
|
||||
attributes in the new instance, so they can be accessed later inside the
|
||||
attributes in the new instance so they can be accessed later inside the
|
||||
spider's code.
|
||||
|
||||
:param crawler: crawler to which the spider will be bound
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ Scrapy uses the `odd-numbered versions for development releases`_.
|
|||
There are 3 numbers in a Scrapy version: *A.B.C*
|
||||
|
||||
* *A* is the major version. This will rarely change and will signify very
|
||||
large changes. So far, only zero is available for *A* as Scrapy hasn't yet
|
||||
reached 1.0.
|
||||
large changes.
|
||||
* *B* is the release number. This will include many changes including features
|
||||
and things that possibly break backwards compatibility. Even Bs will be
|
||||
stable branches, and odd Bs will be development.
|
||||
|
|
@ -27,12 +26,10 @@ For example:
|
|||
API Stability
|
||||
=============
|
||||
|
||||
API stability is one of Scrapy major goals for the *1.0* release, which doesn't
|
||||
have a due date scheduled yet.
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
||||
Methods or functions that start with a single dash (``_``) are private and
|
||||
should never be relied as stable. Besides those, the plan is to stabilize and
|
||||
document the entire API, as we approach the 1.0 release.
|
||||
should never be relied as stable.
|
||||
|
||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||
new methods or functionality but the existing methods should keep working the
|
||||
|
|
|
|||
|
|
@ -15,7 +15,11 @@ from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
|||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}
|
||||
default_settings = {
|
||||
'KEEP_ALIVE': True,
|
||||
'LOGSTATS_INTERVAL': 0,
|
||||
'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter',
|
||||
}
|
||||
|
||||
def syntax(self):
|
||||
return "[url|file]"
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
|
|||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
default_settings = {'LOG_ENABLED': False}
|
||||
|
||||
def syntax(self):
|
||||
return "<project_name>"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ from scrapy.commands import ScrapyCommand
|
|||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
default_settings = {'LOG_ENABLED': False}
|
||||
|
||||
def syntax(self):
|
||||
return "[-v]"
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ class FileDownloadHandler(object):
|
|||
@defers
|
||||
def download_request(self, request, spider):
|
||||
filepath = file_uri_to_path(request.url)
|
||||
body = open(filepath, 'rb').read()
|
||||
with open(filepath, 'rb') as fo:
|
||||
body = fo.read()
|
||||
respcls = responsetypes.from_args(filename=filepath, body=body)
|
||||
return respcls(url=request.url, body=body)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
self.logger = logging.getLogger(__name__)
|
||||
if path:
|
||||
self.file = open(os.path.join(path, 'requests.seen'), 'a+')
|
||||
self.file.seek(0)
|
||||
self.fingerprints.update(x.rstrip() for x in self.file)
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -25,8 +25,10 @@ def xmliter(obj, nodename):
|
|||
- a unicode string
|
||||
- a string encoded as utf-8
|
||||
"""
|
||||
HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
|
||||
HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
|
||||
nodename_patt = re.escape(nodename)
|
||||
|
||||
HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S)
|
||||
HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S)
|
||||
text = _body_or_str(obj)
|
||||
|
||||
header_start = re.search(HEADER_START_RE, text)
|
||||
|
|
@ -34,7 +36,7 @@ def xmliter(obj, nodename):
|
|||
header_end = re_rsearch(HEADER_END_RE, text)
|
||||
header_end = text[header_end[1]:].strip() if header_end else ''
|
||||
|
||||
r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
|
||||
r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
|
||||
for match in r.finditer(text):
|
||||
nodetext = header_start + match.group() + header_end
|
||||
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ Default values
|
|||
|
||||
p['numbers'] # returns []
|
||||
|
||||
Accesing and changing nested item values
|
||||
Accessing and changing nested item values
|
||||
----------------------------------------
|
||||
|
||||
::
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ Request Extractors
|
|||
|
||||
Request Extractors takes response object and determines which requests follow.
|
||||
|
||||
This is an enhancemente to ``LinkExtractors`` which returns urls (links),
|
||||
This is an enhancement to ``LinkExtractors`` which returns urls (links),
|
||||
Request Extractors return Request objects.
|
||||
|
||||
Request Processors
|
||||
|
|
|
|||
|
|
@ -477,7 +477,7 @@ This is a port of the Offsite middleware to the new spider middleware API:
|
|||
|
||||
def should_follow(self, request, spider):
|
||||
info = self.spiders[spider]
|
||||
# hostanme can be None for wrong urls (like javascript links)
|
||||
# hostname can be None for wrong urls (like javascript links)
|
||||
host = urlparse_cached(request).hostname or ''
|
||||
return bool(info.regex.search(host))
|
||||
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ Rationale
|
|||
=========
|
||||
|
||||
There are certain markup patterns that lend themselves quite nicely to
|
||||
automated parsing, for example the ``<table>`` tag outlilnes such a pattern
|
||||
automated parsing, for example the ``<table>`` tag outlines such a pattern
|
||||
for populating a database table with the embedded ``<tr>`` elements denoting
|
||||
the rows and the furthur embedded ``<td>`` elements denoting the individual
|
||||
the rows and the further embedded ``<td>`` elements denoting the individual
|
||||
fields.
|
||||
|
||||
One pattern that is particularly well suited for auto-populating an Item Loader
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import hashlib
|
||||
import tempfile
|
||||
import unittest
|
||||
import shutil
|
||||
|
||||
from scrapy.dupefilters import RFPDupeFilter
|
||||
from scrapy.http import Request
|
||||
|
|
@ -23,6 +25,27 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
dupefilter.close('finished')
|
||||
|
||||
def test_dupefilter_path(self):
|
||||
r1 = Request('http://scrapytest.org/1')
|
||||
r2 = Request('http://scrapytest.org/2')
|
||||
|
||||
path = tempfile.mkdtemp()
|
||||
try:
|
||||
df = RFPDupeFilter(path)
|
||||
df.open()
|
||||
assert not df.request_seen(r1)
|
||||
assert df.request_seen(r1)
|
||||
df.close('finished')
|
||||
|
||||
df2 = RFPDupeFilter(path)
|
||||
df2.open()
|
||||
assert df2.request_seen(r1)
|
||||
assert not df2.request_seen(r2)
|
||||
assert df2.request_seen(r2)
|
||||
df2.close('finished')
|
||||
finally:
|
||||
shutil.rmtree(path)
|
||||
|
||||
def test_request_fingerprint(self):
|
||||
"""Test if customization of request_fingerprint method will change
|
||||
output of request_seen.
|
||||
|
|
|
|||
|
|
@ -33,6 +33,19 @@ class XmliterTestCase(unittest.TestCase):
|
|||
self.assertEqual(attrs,
|
||||
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
|
||||
|
||||
def test_xmliter_unusual_node(self):
|
||||
body = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<root>
|
||||
<matchme...></matchme...>
|
||||
<matchmenot></matchmenot>
|
||||
</root>
|
||||
"""
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
nodenames = [e.xpath('name()').extract()
|
||||
for e in self.xmliter(response, 'matchme...')]
|
||||
self.assertEqual(nodenames, [['matchme...']])
|
||||
|
||||
|
||||
def test_xmliter_text(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue